Selection functions
lookup
Commentary
added in 0.0.5
Retrieves a previously generated event from the supplied collection, optionally drilling into it with a path. This lets you create relationships, through shared identifiers, across different data sets.
ShadowTraffic guarantees that events will only be available for lookup after they have been successfully written to the target collection.
By default, the event chosen from a lookup is entirely random. You can bias how the event is chosen from the population using a histogram - see the examples.
Examples
Kafka lookup
Look up data in another Kafka topic.
{
"generators": [
{
"topic": "a",
"key": {
"id": {
"_gen": "string",
"expr": "#{Name.firstName}"
}
}
},
{
"topic": "b",
"value": {
"_gen": "lookup",
"topic": "a",
"path": [
"key",
"id"
]
}
}
],
"connections": {
"kafka": {
"kind": "kafka",
"producerConfigs": {
"bootstrap.servers": "localhost:9092",
"key.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer",
"value.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer"
}
}
}
}
[
{
"topic": "a",
"key": {
"id": "Genia"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Genia"
},
{
"topic": "a",
"key": {
"id": "Marie"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Marie"
},
{
"topic": "a",
"key": {
"id": "Joey"
},
"value": null
}
]
Postgres lookup
Look up data in a Postgres table.
{
"generators": [
{
"table": "a",
"row": {
"id": {
"_gen": "string",
"expr": "#{Name.firstName}"
}
}
},
{
"table": "b",
"row": {
"id": {
"_gen": "lookup",
"table": "a",
"path": [
"row",
"id"
]
}
}
}
],
"connections": {
"postgres": {
"kind": "postgres",
"connectionConfigs": {
"host": "localhost",
"port": 5432,
"username": "postgres",
"password": "postgres",
"db": "mydb"
}
}
}
}
[
{
"table": "a",
"row": {
"id": "Johnie"
}
},
{
"table": "b",
"row": {
"id": "Johnie"
}
},
{
"table": "a",
"row": {
"id": "Lisa"
}
},
{
"table": "b",
"row": {
"id": "Lisa"
}
},
{
"table": "a",
"row": {
"id": "Woodrow"
}
}
]
Periodic lookups
Sometimes make a new key, sometimes use a previously generated one.
{
"generators": [
{
"topic": "users",
"key": {
"_gen": "weightedOneOf",
"choices": [
{
"weight": 5,
"value": {
"_gen": "string",
"expr": "#{Name.fullName}"
}
},
{
"weight": 5,
"value": {
"_gen": "lookup",
"topic": "users",
"path": [
"key"
]
}
}
]
}
}
],
"connections": {
"kafka": {
"kind": "kafka",
"producerConfigs": {
"bootstrap.servers": "localhost:9092",
"key.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer",
"value.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer"
}
}
}
}
[
{
"topic": "users",
"key": "Carmelita White",
"value": null
},
{
"topic": "users",
"key": "Ms. Alfonso Lueilwitz",
"value": null
},
{
"topic": "users",
"key": "Franklyn Dietrich",
"value": null
},
{
"topic": "users",
"key": "Carmelita White",
"value": null
},
{
"topic": "users",
"key": "Carmelita White",
"value": null
},
{
"topic": "users",
"key": "Carmelita White",
"value": null
},
{
"topic": "users",
"key": "Carmelita White",
"value": null
},
{
"topic": "users",
"key": "Zack Ruecker",
"value": null
},
{
"topic": "users",
"key": "Suzan Frami",
"value": null
},
{
"topic": "users",
"key": "Carmelita White",
"value": null
}
]
Explicit connections
Explicitly supply the connection name when there are multiple connections.
{
"generators": [
{
"connection": "postgres",
"table": "a",
"row": {
"email": {
"_gen": "string",
"expr": "#{Internet.emailAddress}"
}
}
},
{
"connection": "kafka",
"topic": "b",
"value": {
"_gen": "lookup",
"connection": "postgres",
"table": "a",
"path": [
"email"
]
}
}
],
"connections": {
"kafka": {
"kind": "kafka",
"producerConfigs": {
"bootstrap.servers": "localhost:9092",
"key.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer",
"value.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer"
}
},
"postgres": {
"kind": "postgres",
"connectionConfigs": {
"host": "localhost",
"port": 5432,
"username": "postgres",
"password": "postgres",
"db": "mydb"
}
}
}
}
[
{
"table": "a",
"row": {
"email": "dean.durgan@hotmail.com"
}
},
{
"topic": "b",
"key": null,
"value": null
},
{
"table": "a",
"row": {
"email": "virgilio.thiel@hotmail.com"
}
},
{
"topic": "b",
"key": null,
"value": null
},
{
"table": "a",
"row": {
"email": "roderick.mayert@yahoo.com"
}
}
]
Controlling distribution
Use a histogram to control how the element is selected from the population. This invocation chooses 20% of the elements 80% of the time from a Kafka topic.
{
"generators": [
{
"topic": "a",
"key": {
"id": {
"_gen": "string",
"expr": "#{Name.fullName}"
}
}
},
{
"topic": "b",
"value": {
"_gen": "lookup",
"topic": "a",
"path": [
"key",
"id"
],
"histogram": {
"_gen": "histogram",
"bins": [
{
"bin": 0.2,
"frequency": 8
},
{
"bin": 0.8,
"frequency": 2
}
]
}
}
}
],
"connections": {
"kafka": {
"kind": "kafka",
"producerConfigs": {
"bootstrap.servers": "localhost:9092",
"key.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer",
"value.serializer": "io.shadowtraffic.kafka.serdes.JsonSerializer"
}
}
}
}
[
{
"topic": "a",
"key": {
"id": "Miss Millie Bernhard"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Miss Millie Bernhard"
},
{
"topic": "a",
"key": {
"id": "Miss Ben Thompson"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Miss Millie Bernhard"
},
{
"topic": "a",
"key": {
"id": "Elmer Bartell"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Miss Millie Bernhard"
},
{
"topic": "a",
"key": {
"id": "Cora Russel"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Miss Millie Bernhard"
},
{
"topic": "a",
"key": {
"id": "Myron Bartell"
},
"value": null
},
{
"topic": "b",
"key": null,
"value": "Miss Millie Bernhard"
}
]
Specification
JSON schema
Lookups against different connection types have different schemas. Each schema is listed below in array form.
[
{
"name": "Kafka",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"topic": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"required": [
"topic",
"path"
]
}
},
{
"name": "Postgres",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"table": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"required": [
"table",
"path"
]
}
},
{
"name": "Proton",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"stream": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"required": [
"stream",
"path"
]
}
},
{
"name": "Timeplus",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"stream": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"required": [
"stream",
"path"
]
}
},
{
"name": "S3",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"bucket": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"required": [
"bucket",
"path"
]
}
},
{
"name": "EventStore",
"schema": {
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"stream": {
"type": "string"
},
"path": {
"type": "array",
"items": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "string"
}
]
}
}
},
"required": [
"stream",
"path"
]
}
}
]