Skip to main content

Selection functions

lookup

Commentary

added in 0.0.5

Retrieves a previously generated event from the supplied collection, optionally drilling into it with a path. This lets you create relationships, through shared identifiers, across different data sets.

ShadowTraffic guarantees that events will only be available for lookup after they have been successfully written to the target collection.

By default, the event chosen from a lookup is entirely random. But if you like, you can bias how the event is chosen from the population using a histogram. 1


Examples

Kafka lookup

Look up data in another Kafka topic.

{
"generators": [
{
"topic": "a",
"key": {
"id": {
"_gen": "string",
"expr": "#{Name.firstName}"
}
}
},
{
"topic": "b",
"value": {
"_gen": "lookup",
"topic": "a",
"path": [
"key",
"id"
]
}
}
],
"connections": {
"kafka": {
"kind": "kafka",
"producerConfigs": {
"bootstrap.servers": "localhost:9092",
"key.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer",
"value.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer"
}
}
}
}
[
{
"topic": "a",
"key": {
"id": "Maryalice"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Maryalice"
},
{
"topic": "a",
"key": {
"id": "Mary"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Mary"
},
{
"topic": "a",
"key": {
"id": "Connie"
},
"value": null
}
]

Postgres lookup

Look up data in a Postgres table.

{
"generators": [
{
"table": "a",
"row": {
"id": {
"_gen": "string",
"expr": "#{Name.firstName}"
}
}
},
{
"table": "b",
"row": {
"id": {
"_gen": "lookup",
"table": "a",
"path": [
"row",
"id"
]
}
}
}
],
"connections": {
"postgres": {
"kind": "postgres",
"connectionConfigs": {
"host": "localhost",
"port": 5432,
"username": "postgres",
"password": "postgres",
"db": "mydb"
}
}
}
}
[
{
"table": "a",
"row": {
"id": "Keturah"
}
},
{
"table": "b",
"row": {
"id": "Keturah"
}
},
{
"table": "a",
"row": {
"id": "Sharika"
}
},
{
"table": "b",
"row": {
"id": "Keturah"
}
},
{
"table": "a",
"row": {
"id": "Tawanda"
}
}
]

Periodic lookups

Sometimes make a new key, sometimes use a previously generated one.

{
"generators": [
{
"topic": "users",
"key": {
"_gen": "weightedOneOf",
"choices": [
{
"weight": 5,
"value": {
"_gen": "string",
"expr": "#{Name.fullName}"
}
},
{
"weight": 5,
"value": {
"_gen": "lookup",
"topic": "users",
"path": [
"key"
]
}
}
]
}
}
],
"connections": {
"kafka": {
"kind": "kafka",
"producerConfigs": {
"bootstrap.servers": "localhost:9092",
"key.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer",
"value.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer"
}
}
}
}
[
{
"topic": "users",
"key": "Wendell Braun",
"value": null
},
{
"topic": "users",
"key": "Erwin Nader",
"value": null
},
{
"topic": "users",
"key": "Erwin Nader",
"value": null
},
{
"topic": "users",
"key": "Erwin Nader",
"value": null
},
{
"topic": "users",
"key": "Miss Eloy Hickle",
"value": null
},
{
"topic": "users",
"key": "Sharla King",
"value": null
},
{
"topic": "users",
"key": "Wendell Braun",
"value": null
},
{
"topic": "users",
"key": "Christiana Maggio",
"value": null
},
{
"topic": "users",
"key": "Erwin Nader",
"value": null
}
]

Explicit connections

Explicitly supply the connection name when there are multiple connections.

{
"generators": [
{
"connection": "postgres",
"table": "a",
"row": {
"email": {
"_gen": "string",
"expr": "#{Internet.emailAddress}"
}
}
},
{
"connection": "kafka",
"topic": "b",
"value": {
"_gen": "lookup",
"connection": "postgres",
"table": "a",
"path": [
"row",
"email"
]
}
}
],
"connections": {
"kafka": {
"kind": "kafka",
"producerConfigs": {
"bootstrap.servers": "localhost:9092",
"key.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer",
"value.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer"
}
},
"postgres": {
"kind": "postgres",
"connectionConfigs": {
"host": "localhost",
"port": 5432,
"username": "postgres",
"password": "postgres",
"db": "mydb"
}
}
}
}
[
{
"table": "a",
"row": {
"email": "drew.mills@gmail.com"
}
},
{
"topic": "b",
"key": null,
"value": "drew.mills@gmail.com"
},
{
"table": "a",
"row": {
"email": "tyler.white@gmail.com"
}
},
{
"topic": "b",
"key": null,
"value": "drew.mills@gmail.com"
},
{
"table": "a",
"row": {
"email": "nigel.reynolds@gmail.com"
}
}
]

Controlling distribution

Use a histogram to control how the element is selected from the population. This invocation chooses 20% of the elements 80% of the time from a Kafka topic.

{
"generators": [
{
"topic": "a",
"key": {
"id": {
"_gen": "string",
"expr": "#{Name.fullName}"
}
}
},
{
"topic": "b",
"value": {
"_gen": "lookup",
"topic": "a",
"path": [
"key",
"id"
],
"histogram": {
"_gen": "histogram",
"bins": [
{
"bin": 0.2,
"frequency": 8
},
{
"bin": 0.8,
"frequency": 2
}
]
}
}
}
],
"connections": {
"kafka": {
"kind": "kafka",
"producerConfigs": {
"bootstrap.servers": "localhost:9092",
"key.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer",
"value.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer"
}
}
}
}
[
{
"topic": "a",
"key": {
"id": "Dale Fahey"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Dale Fahey"
},
{
"topic": "a",
"key": {
"id": "Hollis Bednar PhD"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Dale Fahey"
},
{
"topic": "a",
"key": {
"id": "Alexander Gorczany"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Dale Fahey"
},
{
"topic": "a",
"key": {
"id": "Jarred Bergnaum"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Dale Fahey"
},
{
"topic": "a",
"key": {
"id": "Rubi Shanahan"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Dale Fahey"
}
]

Multiple lookups

If you need to look up multiple fields, be careful not to use lookup more than once in the same generator. Multiple calls are not consistent. In other words, they won't return the same event in each call.

Instead, call lookup just once by using a variable and setting path to [] to grab then entire event. Then pick out the relevant fields.

[
{
"topic": "a",
"key": {
"name": {
"_gen": "string",
"expr": "#{Name.fullName}"
},
"magicNumber": {
"_gen": "uniformDistribution",
"bounds": [
0,
100
],
"decimals": 0
}
}
},
{
"topic": "b",
"vars": {
"result": {
"_gen": "lookup",
"topic": "a",
"path": []
}
},
"value": {
"lookedUpName": {
"_gen": "var",
"var": "result",
"path": [
"key",
"name"
]
},
"lookedUpNumber": {
"_gen": "var",
"var": "result",
"path": [
"key",
"magicNumber"
]
}
}
}
]
[
{
"topic": "a",
"key": {
"name": "Miss Millie Bernhard",
"magicNumber": 80
},
"value": null
},
{
"topic": "b",
"key": null,
"value": {
"lookedUpName": "Miss Millie Bernhard",
"lookedUpNumber": 80
}
},
{
"topic": "a",
"key": {
"name": "Alysa Cormier",
"magicNumber": 50
},
"value": null
},
{
"topic": "b",
"key": null,
"value": {
"lookedUpName": "Alysa Cormier",
"lookedUpNumber": 50
}
},
{
"topic": "a",
"key": {
"name": "Tori Keebler",
"magicNumber": 61
},
"value": null
},
{
"topic": "b",
"key": null,
"value": {
"lookedUpName": "Alysa Cormier",
"lookedUpNumber": 50
}
},
{
"topic": "a",
"key": {
"name": "Mr. Myron Bartell",
"magicNumber": 3
},
"value": null
},
{
"topic": "b",
"key": null,
"value": {
"lookedUpName": "Miss Millie Bernhard",
"lookedUpNumber": 80
}
},
{
"topic": "a",
"key": {
"name": "Brunilda Macejkovic DVM",
"magicNumber": 17
},
"value": null
},
{
"topic": "b",
"key": null,
"value": {
"lookedUpName": "Tori Keebler",
"lookedUpNumber": 61
}
}
]

Specification

JSON schema

Lookups against different connection types have different schemas. Each schema is listed below in array form.

[
{
"name": "Kafka",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"topic": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"required": [
"topic",
"path"
]
}
},
{
"name": "Postgres",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"table": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"required": [
"table",
"path"
]
}
},
{
"name": "Proton",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"stream": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"required": [
"stream",
"path"
]
}
},
{
"name": "Timeplus",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"stream": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"required": [
"stream",
"path"
]
}
},
{
"name": "S3",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"bucket": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"required": [
"bucket",
"path"
]
}
},
{
"name": "fileSystem",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"fileName": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"required": [
"fileName",
"path"
]
}
},
{
"name": "EventStore",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"stream": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"required": [
"stream",
"path"
]
}
}
]