filesystem

Commentary

added in 0.5.10

Writes to the local file system.

Specify the path to write to in the connection map. If any of the parent directories don't exist, they will automatically be created. ¹

ShadowTraffic will write a series of files to that path, periodically rolling a new one. Specify the base file name with fileName, and files will be generated in the pattern fileName-n.suffix. The value of n will always increase on the latest file generated. ²

A new file will be created following the default batch rate, which can be overriden by time, elements, or serialized bytes. ³

You can choose from a range of serialization formats and compression types ⁴.

Do not write multiple generators that output to the same file name in the same directory, since the output writers will contend with each other and overwrite previously written content.

Formats

Set the data format with the fileConfig parameter. json and jsonl are currently supported.

---

Examples

Configuring the connection

At minimum, use path specify to where the files should be written.

{
    "connections": {
        "localFs": {
            "kind": "fileSystem",
            "path": "/tmp/data"
        }
    }
}

Writing to files

Use fileName to set the base file name to be written. In this example, files named transactions-0.json, transactions-n.json, ... will be created. Use data to set the data to be written to the file.

{
    "generators": [
        {
            "fileName": "transactions",
            "fileConfigs": {
                "format": "json"
            },
            "data": {
                "amount": {
                    "_gen": "normalDistribution",
                    "mean": 100,
                    "sd": 2
                }
            }
        }
    ],
    "connections": {
        "localFs": {
            "kind": "fileSystem",
            "path": "/tmp/data"
        }
    }
}

Set the batch rate

By default, a new file will be rolled every 500 ms or 5000 elements, whichever happens first. You can also optionally roll a new file after a certain amount of serialized bytes have been accumulated.

To override these:

• use lingerMs to set the limit on time
• use batchElements to set it on number of events
• use batchBytes to set it on size

{
    "connections": {
        "localFs": {
            "kind": "fileSystem",
            "path": "/tmp/data",
            "batchConfigs": {
                "lingerMs": 5000,
                "batchElements": 50000,
                "batchBytes": 5242880
            }
        }
    }
}

Set the format and compression

format can be any of json, jsonl, and parquet. Additionally, pretty set to true will cause json to pretty print.

compression can optionally be set to gzip.

{
    "generators": [
        {
            "fileName": "ipAddresses",
            "fileConfigs": {
                "format": "json",
                "pretty": true
            },
            "data": {
                "ip": {
                    "_gen": "string",
                    "expr": "#{Internet.ipV4Address}"
                },
                "timestamp": {
                    "_now": "now"
                }
            }
        }
    ],
    "connections": {
        "localFs": {
            "kind": "fileSystem",
            "path": "/tmp/data"
        }
    }
}

---

Specification

Connection JSON schema

{
    "type": "object",
    "properties": {
        "kind": {
            "type": "string",
            "const": "fileSystem"
        },
        "path": {
            "type": "string"
        },
        "batchConfigs": {
            "type": "object",
            "properties": {
                "lingerMs": {
                    "type": "integer",
                    "minimum": 0
                },
                "batchElements": {
                    "type": "integer",
                    "minimum": 1
                },
                "batchBytes": {
                    "type": "integer",
                    "minimum": 1
                }
            }
        }
    },
    "required": [
        "path"
    ]
}

Generator JSON schema

{
    "type": "object",
    "properties": {
        "connection": {
            "type": "string"
        },
        "name": {
            "type": "string"
        },
        "fileName": {
            "type": "string"
        },
        "data": {},
        "localConfigs": {
            "type": "object",
            "properties": {
                "throttleMs": {
                    "oneOf": [
                        {
                            "type": "number",
                            "minimum": 0
                        },
                        {
                            "type": "object",
                            "properties": {
                                "_gen": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "_gen"
                            ]
                        }
                    ]
                },
                "maxEvents": {
                    "oneOf": [
                        {
                            "type": "integer",
                            "minimum": 0
                        },
                        {
                            "type": "object",
                            "properties": {
                                "_gen": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "_gen"
                            ]
                        }
                    ]
                },
                "kafkaKeyProtobufHint": {
                    "type": "object",
                    "properties": {
                        "schemaFile": {
                            "type": "string"
                        },
                        "message": {
                            "type": "string"
                        }
                    },
                    "required": [
                        "schemaFile",
                        "message"
                    ]
                },
                "jsonSchemaHint": {
                    "type": "object"
                },
                "maxBytes": {
                    "type": "integer",
                    "minimum": 1
                },
                "discard": {
                    "type": "object",
                    "properties": {
                        "rate": {
                            "type": "number",
                            "minimum": 0,
                            "maximum": 1
                        }
                    },
                    "required": [
                        "rate"
                    ]
                },
                "repeat": {
                    "type": "object",
                    "properties": {
                        "rate": {
                            "type": "number",
                            "minimum": 0,
                            "maximum": 1
                        },
                        "times": {
                            "oneOf": [
                                {
                                    "type": "integer",
                                    "minimum": 0
                                },
                                {
                                    "type": "object",
                                    "properties": {
                                        "_gen": {
                                            "type": "string"
                                        }
                                    },
                                    "required": [
                                        "_gen"
                                    ]
                                }
                            ]
                        }
                    },
                    "required": [
                        "rate",
                        "times"
                    ]
                },
                "protobufSchemaHint": {
                    "type": "object",
                    "patternProperties": {
                        "^.*$": {
                            "type": "object",
                            "properties": {
                                "schemaFile": {
                                    "type": "string"
                                },
                                "message": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "schemaFile",
                                "message"
                            ]
                        }
                    }
                },
                "maxHistoryEvents": {
                    "type": "integer",
                    "minimum": 0
                },
                "maxMs": {
                    "type": "integer",
                    "minimum": 0
                },
                "time": {
                    "type": "integer"
                },
                "events": {
                    "type": "object",
                    "properties": {
                        "exactly": {
                            "oneOf": [
                                {
                                    "type": "integer",
                                    "minimum": 0
                                },
                                {
                                    "type": "object",
                                    "properties": {
                                        "_gen": {
                                            "type": "string"
                                        }
                                    },
                                    "required": [
                                        "_gen"
                                    ]
                                }
                            ]
                        }
                    }
                },
                "delay": {
                    "type": "object",
                    "properties": {
                        "rate": {
                            "type": "number",
                            "minimum": 0,
                            "maximum": 1
                        },
                        "ms": {
                            "oneOf": [
                                {
                                    "type": "integer",
                                    "minimum": 0
                                },
                                {
                                    "type": "object",
                                    "properties": {
                                        "_gen": {
                                            "type": "string"
                                        }
                                    },
                                    "required": [
                                        "_gen"
                                    ]
                                }
                            ]
                        }
                    },
                    "required": [
                        "rate",
                        "ms"
                    ]
                },
                "history": {
                    "type": "object",
                    "properties": {
                        "events": {
                            "type": "object",
                            "properties": {
                                "max": {
                                    "type": "integer",
                                    "minimum": 0
                                }
                            }
                        }
                    }
                },
                "avroSchemaHint": {
                    "type": "object"
                },
                "throttle": {
                    "type": "object",
                    "properties": {
                        "ms": {
                            "oneOf": [
                                {
                                    "type": "number",
                                    "minimum": 0
                                },
                                {
                                    "type": "object",
                                    "properties": {
                                        "_gen": {
                                            "type": "string"
                                        }
                                    },
                                    "required": [
                                        "_gen"
                                    ]
                                }
                            ]
                        }
                    }
                },
                "throughput": {
                    "oneOf": [
                        {
                            "type": "integer",
                            "minimum": 1
                        },
                        {
                            "type": "object",
                            "properties": {
                                "_gen": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "_gen"
                            ]
                        }
                    ]
                },
                "timeMultiplier": {
                    "oneOf": [
                        {
                            "type": "number"
                        },
                        {
                            "type": "object",
                            "properties": {
                                "_gen": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "_gen"
                            ]
                        }
                    ]
                },
                "kafkaValueProtobufHint": {
                    "type": "object",
                    "properties": {
                        "schemaFile": {
                            "type": "string"
                        },
                        "message": {
                            "type": "string"
                        }
                    },
                    "required": [
                        "schemaFile",
                        "message"
                    ]
                }
            }
        },
        "fileConfigs": {
            "type": "object",
            "properties": {
                "format": {
                    "type": "string",
                    "enum": [
                        "json",
                        "jsonl",
                        "parquet"
                    ]
                },
                "pretty": {
                    "type": "boolean"
                },
                "compression": {
                    "type": "string",
                    "enum": [
                        "gzip"
                    ]
                }
            },
            "required": [
                "format"
            ]
        }
    },
    "required": [
        "fileName",
        "data",
        "fileConfigs"
    ]
}