Skip to main content

Connections

s3

Commentary​

added in 0.2.0

Connects to Amazon S3. Credentials are read the same way as the AWS Java client: set environment variables for AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and AWS_SESSION_TOKEN. Region is also set the same way, with the environment variable. AWS_REGION.

Target bucket must exist prior to writing data.

A new object is created every 500 ms or 5000 elements, whichever happens first. You can override object size/write timing with batchConfigs.

This can also be used to connect to MinIO or similar by supplying endpoint in the connectionConfigs.

Examples​

Configuring the connection​

{
    "connections": {
        "s3-staging-org": {
            "kind": "s3",
            "batchConfigs": {
                "lingerMs": 2000,
                "batchElements": 10000
            }
        }
    }
}

Specification​

Connection JSON schema​

{
    "type": "object",
    "properties": {
        "kind": {
            "type": "string",
            "const": "s3"
        },
        "batchConfigs": {
            "type": "object",
            "properties": {
                "lingerMs": {
                    "type": "integer",
                    "minimum": 0
                },
                "batchElements": {
                    "type": "integer",
                    "minimum": 1
                }
            }
        },
        "connectionConfigs": {
            "type": "object",
            "properties": {
                "endpoint": {
                    "type": "string"
                }
            }
        }
    }
}

Generator JSON schema​

{
    "type": "object",
    "properties": {
        "connection": {
            "type": "string"
        },
        "bucket": {
            "type": "string"
        },
        "data": {
            "type": "object"
        },
        "localConfigs": {
            "type": "object",
            "properties": {
                "maxEvents": {
                    "oneOf": [
                        {
                            "type": "integer",
                            "minimum": 0
                        },
                        {
                            "type": "object",
                            "properties": {
                                "_gen": {
                                    "type": "string"
                                }
                            },
                            "required": [
                                "_gen"
                            ]
                        }
                    ]
                },
                "events": {
                    "type": "object",
                    "properties": {
                        "exactly": {
                            "oneOf": [
                                {
                                    "type": "integer",
                                    "minimum": 0
                                },
                                {
                                    "type": "object",
                                    "properties": {
                                        "_gen": {
                                            "type": "string"
                                        }
                                    },
                                    "required": [
                                        "_gen"
                                    ]
                                }
                            ]
                        }
                    }
                },
                "throttle": {
                    "type": "object",
                    "properties": {
                        "ms": {
                            "oneOf": [
                                {
                                    "type": "number",
                                    "minimum": 0
                                },
                                {
                                    "type": "object"
                                }
                            ]
                        }
                    }
                },
                "history": {
                    "type": "object",
                    "properties": {
                        "events": {
                            "type": "object",
                            "properties": {
                                "max": {
                                    "type": "integer",
                                    "minimum": 0
                                }
                            }
                        }
                    }
                },
                "discard": {
                    "type": "object",
                    "properties": {
                        "rate": {
                            "type": "number",
                            "minimum": 0,
                            "maximum": 1
                        }
                    },
                    "required": [
                        "rate"
                    ]
                },
                "repeat": {
                    "type": "object",
                    "properties": {
                        "rate": {
                            "type": "number",
                            "minimum": 0,
                            "maximum": 1
                        },
                        "times": {
                            "oneOf": [
                                {
                                    "type": "integer",
                                    "minimum": 0
                                },
                                {
                                    "type": "object",
                                    "properties": {
                                        "_gen": {
                                            "type": "string"
                                        }
                                    },
                                    "required": [
                                        "_gen"
                                    ]
                                }
                            ]
                        }
                    },
                    "required": [
                        "rate",
                        "times"
                    ]
                },
                "delay": {
                    "type": "object",
                    "properties": {
                        "rate": {
                            "type": "number",
                            "minimum": 0,
                            "maximum": 1
                        },
                        "ms": {
                            "oneOf": [
                                {
                                    "type": "integer",
                                    "minimum": 0
                                },
                                {
                                    "type": "object",
                                    "properties": {
                                        "_gen": {
                                            "type": "string"
                                        }
                                    },
                                    "required": [
                                        "_gen"
                                    ]
                                }
                            ]
                        }
                    },
                    "required": [
                        "rate",
                        "ms"
                    ]
                }
            }
        },
        "bucketConfigs": {
            "type": "object",
            "properties": {
                "keyPrefix": {
                    "type": "string"
                },
                "format": {
                    "type": "string",
                    "enum": [
                        "jsonl"
                    ]
                }
            },
            "required": [
                "keyPrefix",
                "format"
            ]
        }
    },
    "required": [
        "bucket",
        "data",
        "bucketConfigs"
    ]
}