Selection functions
lookup
Commentary
added in 0.0.5
Retrieves a previously generated event from the supplied collection, optionally drilling into it with a path. This lets you create relationships, through shared identifiers, across different data sets.
ShadowTraffic guarantees that events will only be available for lookup after they have been successfully written to the target collection.
When looking up data, you can either target a generator by its output target 1 or its generator name. 2
By default, the event chosen from a lookup is entirely random. But if you like, you can bias how the event is chosen from the population using a histogram. 3
Examples
Kafka lookup
Look up data in another Kafka topic.
{
"generators": [
{
"topic": "a",
"key": {
"id": {
"_gen": "string",
"expr": "#{Name.firstName}"
}
}
},
{
"topic": "b",
"value": {
"_gen": "lookup",
"topic": "a",
"path": [
"key",
"id"
]
}
}
],
"connections": {
"kafka": {
"kind": "kafka",
"producerConfigs": {
"bootstrap.servers": "localhost:9092",
"key.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer",
"value.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer"
}
}
}
}
[
{
"topic": "a",
"key": {
"id": "Maurine"
},
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": "Maurine",
"headers": null
},
{
"topic": "a",
"key": {
"id": "Nathanael"
},
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": "Maurine",
"headers": null
},
{
"topic": "a",
"key": {
"id": "Tiesha"
},
"value": null,
"headers": null
}
]
Postgres lookup
Look up data in a Postgres table.
{
"generators": [
{
"table": "a",
"row": {
"id": {
"_gen": "string",
"expr": "#{Name.firstName}"
}
}
},
{
"table": "b",
"row": {
"id": {
"_gen": "lookup",
"table": "a",
"path": [
"row",
"id"
]
}
}
}
],
"connections": {
"postgres": {
"kind": "postgres",
"connectionConfigs": {
"host": "localhost",
"port": 5432,
"username": "postgres",
"password": "postgres",
"db": "mydb"
}
}
}
}
[
{
"table": "a",
"row": {
"id": "Amalia"
},
"op": null,
"where": null
},
{
"table": "b",
"row": {
"id": "Amalia"
},
"op": null,
"where": null
},
{
"table": "a",
"row": {
"id": "Waylon"
},
"op": null,
"where": null
},
{
"table": "b",
"row": {
"id": "Amalia"
},
"op": null,
"where": null
},
{
"table": "a",
"row": {
"id": "Roseline"
},
"op": null,
"where": null
}
]
Periodic lookups
Sometimes make a new key, sometimes use a previously generated one.
{
"generators": [
{
"topic": "users",
"key": {
"_gen": "weightedOneOf",
"choices": [
{
"weight": 5,
"value": {
"_gen": "string",
"expr": "#{Name.fullName}"
}
},
{
"weight": 5,
"value": {
"_gen": "lookup",
"topic": "users",
"path": [
"key"
]
}
}
]
}
}
],
"connections": {
"kafka": {
"kind": "kafka",
"producerConfigs": {
"bootstrap.servers": "localhost:9092",
"key.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer",
"value.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer"
}
}
}
}
[
{
"topic": "users",
"key": "Mrs. Cleveland Torp",
"value": null,
"headers": null
},
{
"topic": "users",
"key": "Patria Ritchie",
"value": null,
"headers": null
},
{
"topic": "users",
"key": "Patria Ritchie",
"value": null,
"headers": null
},
{
"topic": "users",
"key": "Deon Lind",
"value": null,
"headers": null
},
{
"topic": "users",
"key": "Patria Ritchie",
"value": null,
"headers": null
},
{
"topic": "users",
"key": "Patria Ritchie",
"value": null,
"headers": null
},
{
"topic": "users",
"key": "Gene Gaylord",
"value": null,
"headers": null
},
{
"topic": "users",
"key": "Debi Rohan",
"value": null,
"headers": null
},
{
"topic": "users",
"key": "Leroy Schmitt",
"value": null,
"headers": null
},
{
"topic": "users",
"key": "Patria Ritchie",
"value": null,
"headers": null
}
]
Explicit connections
Explicitly supply the connection name when there are multiple connections.
{
"generators": [
{
"connection": "postgres",
"table": "a",
"row": {
"email": {
"_gen": "string",
"expr": "#{Internet.emailAddress}"
}
}
},
{
"connection": "kafka",
"topic": "b",
"value": {
"_gen": "lookup",
"connection": "postgres",
"table": "a",
"path": [
"row",
"email"
]
}
}
],
"connections": {
"kafka": {
"kind": "kafka",
"producerConfigs": {
"bootstrap.servers": "localhost:9092",
"key.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer",
"value.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer"
}
},
"postgres": {
"kind": "postgres",
"connectionConfigs": {
"host": "localhost",
"port": 5432,
"username": "postgres",
"password": "postgres",
"db": "mydb"
}
}
}
}
[
{
"table": "a",
"row": {
"email": "jean.dickens@yahoo.com"
},
"op": null,
"where": null
},
{
"topic": "b",
"key": null,
"value": "jean.dickens@yahoo.com",
"headers": null
},
{
"table": "a",
"row": {
"email": "emmett.bailey@yahoo.com"
},
"op": null,
"where": null
},
{
"topic": "b",
"key": null,
"value": "emmett.bailey@yahoo.com",
"headers": null
},
{
"table": "a",
"row": {
"email": "jon.sawayn@hotmail.com"
},
"op": null,
"where": null
}
]
Controlling distribution
Use a histogram to control how the element is selected from the population. This invocation chooses 20% of the elements 80% of the time from a Kafka topic.
{
"generators": [
{
"topic": "a",
"key": {
"id": {
"_gen": "string",
"expr": "#{Name.fullName}"
}
}
},
{
"topic": "b",
"value": {
"_gen": "lookup",
"topic": "a",
"path": [
"key",
"id"
],
"histogram": {
"_gen": "histogram",
"bins": [
{
"bin": 0.2,
"frequency": 8
},
{
"bin": 0.8,
"frequency": 2
}
]
}
}
}
],
"connections": {
"kafka": {
"kind": "kafka",
"producerConfigs": {
"bootstrap.servers": "localhost:9092",
"key.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer",
"value.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer"
}
}
}
}
[
{
"topic": "a",
"key": {
"id": "Lynn Parisian"
},
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": "Lynn Parisian",
"headers": null
},
{
"topic": "a",
"key": {
"id": "Amado Mitchell"
},
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": "Lynn Parisian",
"headers": null
},
{
"topic": "a",
"key": {
"id": "Dr. Theodore Hackett"
},
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": "Lynn Parisian",
"headers": null
},
{
"topic": "a",
"key": {
"id": "Leesa Wilkinson II"
},
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": "Lynn Parisian",
"headers": null
},
{
"topic": "a",
"key": {
"id": "Felecia Kreiger PhD"
},
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": "Lynn Parisian",
"headers": null
}
]
Multiple lookups
If you need to look up multiple fields, be careful not to use lookup more than once in the same generator. Multiple calls are not consistent. In other words, they won't return the same event in each call.
Instead, call lookup just once by using a variable and setting path to [] to grab then entire event. Then pick out the relevant fields.
[
{
"topic": "a",
"key": {
"name": {
"_gen": "string",
"expr": "#{Name.fullName}"
},
"magicNumber": {
"_gen": "uniformDistribution",
"bounds": [
0,
100
],
"decimals": 0
}
}
},
{
"topic": "b",
"vars": {
"result": {
"_gen": "lookup",
"topic": "a",
"path": []
}
},
"value": {
"lookedUpName": {
"_gen": "var",
"var": "result",
"path": [
"key",
"name"
]
},
"lookedUpNumber": {
"_gen": "var",
"var": "result",
"path": [
"key",
"magicNumber"
]
}
}
}
]
[
{
"topic": "a",
"key": {
"name": "Beaulah Kemmer",
"magicNumber": 66
},
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": {
"lookedUpName": "Beaulah Kemmer",
"lookedUpNumber": 66
},
"headers": null
},
{
"topic": "a",
"key": {
"name": "Berry Wunsch",
"magicNumber": 5
},
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": {
"lookedUpName": "Berry Wunsch",
"lookedUpNumber": 5
},
"headers": null
},
{
"topic": "a",
"key": {
"name": "Verona Smitham Jr.",
"magicNumber": 53
},
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": {
"lookedUpName": "Beaulah Kemmer",
"lookedUpNumber": 66
},
"headers": null
},
{
"topic": "a",
"key": {
"name": "Mrs. Rashad Torp",
"magicNumber": 15
},
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": {
"lookedUpName": "Beaulah Kemmer",
"lookedUpNumber": 66
},
"headers": null
},
{
"topic": "a",
"key": {
"name": "Neta Reilly",
"magicNumber": 78
},
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": {
"lookedUpName": "Mrs. Rashad Torp",
"lookedUpNumber": 15
},
"headers": null
}
]
Lookup by name
Use name in the lookup function to target another generator by its name attribute. This is useful if multiple generators write to the same output connection and collection and you want to distinguish between them.
[
{
"name": "generator-A",
"topic": "a",
"key": {
"_gen": "uniformDistribution",
"bounds": [
0,
100
],
"decimals": 0
}
},
{
"topic": "b",
"value": {
"result": {
"_gen": "lookup",
"name": "generator-A",
"path": [
"key"
]
}
}
}
]
[
{
"topic": "a",
"key": 56,
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": {
"result": 56
},
"headers": null
},
{
"topic": "a",
"key": 92,
"value": null,
"headers": null
},
{
"topic": "b",
"key": null,
"value": {
"result": 56
},
"headers": null
},
{
"topic": "a",
"key": 44,
"value": null,
"headers": null
}
]
Specification
JSON schema
Lookups against different connection types have different schemas. Each schema is listed below in array form.
[
{
"name": "Kafka",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"topic": {
"type": "string"
},
"name": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"oneOf": [
{
"required": [
"topic",
"path"
]
},
{
"required": [
"name",
"path"
]
}
]
}
},
{
"name": "Postgres",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"table": {
"type": "string"
},
"name": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"oneOf": [
{
"required": [
"table",
"path"
]
},
{
"required": [
"name",
"path"
]
}
]
}
},
{
"name": "sqlServer",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"table": {
"type": "string"
},
"name": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"oneOf": [
{
"required": [
"table",
"path"
]
},
{
"required": [
"name",
"path"
]
}
]
}
},
{
"name": "Proton",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"stream": {
"type": "string"
},
"name": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"oneOf": [
{
"required": [
"stream",
"path"
]
},
{
"required": [
"name",
"path"
]
}
]
}
},
{
"name": "Timeplus",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"stream": {
"type": "string"
},
"name": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"oneOf": [
{
"required": [
"stream",
"path"
]
},
{
"required": [
"name",
"path"
]
}
]
}
},
{
"name": "S3",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"bucket": {
"type": "string"
},
"keyPrefix": {
"type": "string"
},
"name": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"oneOf": [
{
"required": [
"bucket",
"path"
]
},
{
"required": [
"name",
"path"
]
}
]
}
},
{
"name": "googleCloudStorage",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"bucket": {
"type": "string"
},
"blobPrefix": {
"type": "string"
},
"name": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"oneOf": [
{
"required": [
"bucket",
"path"
]
},
{
"required": [
"name",
"path"
]
}
]
}
},
{
"name": "azureBlobStorage",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"container": {
"type": "string"
},
"keyPrefix": {
"type": "string"
},
"name": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"oneOf": [
{
"required": [
"container",
"path"
]
},
{
"required": [
"name",
"path"
]
}
]
}
},
{
"name": "fileSystem",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"fileName": {
"type": "string"
},
"name": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"oneOf": [
{
"required": [
"fileName",
"path"
]
},
{
"required": [
"name",
"path"
]
}
]
}
},
{
"name": "EventStore",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"stream": {
"type": "string"
},
"name": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"oneOf": [
{
"required": [
"stream",
"path"
]
},
{
"required": [
"name",
"path"
]
}
]
}
},
{
"name": "Webhook",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"url": {
"type": "string"
},
"name": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"oneOf": [
{
"required": [
"url",
"path"
]
},
{
"required": [
"name",
"path"
]
}
]
}
}
]