Connections
filesystem
Commentary
added in 0.5.10
Writes to the local file system.
Specify the path to write to in the connection map. If any of the parent directories don't exist, they will automatically be created. 1
ShadowTraffic will write a series of files to that path, periodically rolling a new one. Specify the base file name with fileName, and files will be generated in the pattern fileName-n.suffix. The value of n will always increase on the latest file generated. 2
A new file will be created following the default batch rate, which can be overriden by time, elements, or serialized bytes. 3
You can choose from a range of serialization formats and compression types 4.
Do not write multiple generators that output to the same file name in the same directory, since the output writers will contend with each other and overwrite previously written content.
Formats
Set the data format with the fileConfig parameter. json and jsonl are currently supported.
Examples
Configuring the connection
At minimum, use path specify to where the files should be written.
{
"connections": {
"localFs": {
"kind": "fileSystem",
"path": "/tmp/data"
}
}
}
Writing to files
Use fileName to set the base file name to be written. In this example, files named transactions-0.json, transactions-n.json, ... will be created. Use data to set the data to be written to the file.
{
"generators": [
{
"fileName": "transactions",
"fileConfigs": {
"format": "json"
},
"data": {
"amount": {
"_gen": "normalDistribution",
"mean": 100,
"sd": 2
}
}
}
],
"connections": {
"localFs": {
"kind": "fileSystem",
"path": "/tmp/data"
}
}
}
Set the batch rate
By default, a new file will be rolled every 500 ms or 5000 elements, whichever happens first. You can also optionally roll a new file after a certain amount of serialized bytes have been accumulated.
To override these:
- use
lingerMsto set the limit on time - use
batchElementsto set it on number of events - use
batchBytesto set it on size
{
"connections": {
"localFs": {
"kind": "fileSystem",
"path": "/tmp/data",
"batchConfigs": {
"lingerMs": 5000,
"batchElements": 50000,
"batchBytes": 5242880
}
}
}
}
Set the format and compression
format can be any of json, jsonl, and parquet. Additionally, pretty set to true will cause json to pretty print.
compression can optionally be set to gzip.
{
"generators": [
{
"fileName": "ipAddresses",
"fileConfigs": {
"format": "json",
"pretty": true
},
"data": {
"ip": {
"_gen": "string",
"expr": "#{Internet.ipV4Address}"
},
"timestamp": {
"_now": "now"
}
}
}
],
"connections": {
"localFs": {
"kind": "fileSystem",
"path": "/tmp/data"
}
}
}
Specification
Connection JSON schema
{
"type": "object",
"properties": {
"kind": {
"type": "string",
"const": "fileSystem"
},
"path": {
"type": "string"
},
"batchConfigs": {
"type": "object",
"properties": {
"lingerMs": {
"type": "integer",
"minimum": 0
},
"batchElements": {
"type": "integer",
"minimum": 1
},
"batchBytes": {
"type": "integer",
"minimum": 1
}
}
}
},
"required": [
"path"
]
}
Generator JSON schema
{
"type": "object",
"properties": {
"connection": {
"type": "string"
},
"name": {
"type": "string"
},
"fileName": {
"type": "string"
},
"data": {},
"localConfigs": {
"type": "object",
"properties": {
"throttleMs": {
"oneOf": [
{
"type": "number",
"minimum": 0
},
{
"type": "object",
"properties": {
"_gen": {
"type": "string"
}
},
"required": [
"_gen"
]
}
]
},
"maxEvents": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "object",
"properties": {
"_gen": {
"type": "string"
}
},
"required": [
"_gen"
]
}
]
},
"kafkaKeyProtobufHint": {
"type": "object",
"properties": {
"schemaFile": {
"type": "string"
},
"message": {
"type": "string"
}
},
"required": [
"schemaFile",
"message"
]
},
"jsonSchemaHint": {
"type": "object"
},
"maxBytes": {
"type": "integer",
"minimum": 1
},
"discard": {
"type": "object",
"properties": {
"rate": {
"type": "number",
"minimum": 0,
"maximum": 1
}
},
"required": [
"rate"
]
},
"repeat": {
"type": "object",
"properties": {
"rate": {
"type": "number",
"minimum": 0,
"maximum": 1
},
"times": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "object",
"properties": {
"_gen": {
"type": "string"
}
},
"required": [
"_gen"
]
}
]
}
},
"required": [
"rate",
"times"
]
},
"maxHistoryEvents": {
"type": "integer",
"minimum": 0
},
"maxMs": {
"type": "integer",
"minimum": 0
},
"time": {
"type": "integer"
},
"events": {
"type": "object",
"properties": {
"exactly": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "object",
"properties": {
"_gen": {
"type": "string"
}
},
"required": [
"_gen"
]
}
]
}
}
},
"delay": {
"type": "object",
"properties": {
"rate": {
"type": "number",
"minimum": 0,
"maximum": 1
},
"ms": {
"oneOf": [
{
"type": "integer",
"minimum": 0
},
{
"type": "object",
"properties": {
"_gen": {
"type": "string"
}
},
"required": [
"_gen"
]
}
]
}
},
"required": [
"rate",
"ms"
]
},
"history": {
"type": "object",
"properties": {
"events": {
"type": "object",
"properties": {
"max": {
"type": "integer",
"minimum": 0
}
}
}
}
},
"avroSchemaHint": {
"type": "object"
},
"throttle": {
"type": "object",
"properties": {
"ms": {
"oneOf": [
{
"type": "number",
"minimum": 0
},
{
"type": "object",
"properties": {
"_gen": {
"type": "string"
}
},
"required": [
"_gen"
]
}
]
}
}
},
"throughput": {
"oneOf": [
{
"type": "integer",
"minimum": 1
},
{
"type": "object",
"properties": {
"_gen": {
"type": "string"
}
},
"required": [
"_gen"
]
}
]
},
"timeMultiplier": {
"oneOf": [
{
"type": "number"
},
{
"type": "object",
"properties": {
"_gen": {
"type": "string"
}
},
"required": [
"_gen"
]
}
]
},
"kafkaValueProtobufHint": {
"type": "object",
"properties": {
"schemaFile": {
"type": "string"
},
"message": {
"type": "string"
}
},
"required": [
"schemaFile",
"message"
]
}
}
},
"fileConfigs": {
"type": "object",
"properties": {
"format": {
"type": "string",
"enum": [
"json",
"jsonl",
"parquet"
]
},
"pretty": {
"type": "boolean"
},
"compression": {
"type": "string",
"enum": [
"gzip"
]
}
},
"required": [
"format"
]
}
},
"required": [
"fileName",
"data",
"fileConfigs"
]
}