{ "name": "cloudflare_logpush", "title": "Cloudflare Logpush", "version": "1.44.0", "release": "ga", "description": "Collect and parse logs from Cloudflare API with Elastic Agent.", "type": "integration", "download": "/epr/cloudflare_logpush/cloudflare_logpush-1.44.0.zip", "path": "/package/cloudflare_logpush/1.44.0", "icons": [ { "src": "/img/cloudflare-logo.svg", "path": "/package/cloudflare_logpush/1.44.0/img/cloudflare-logo.svg", "title": "Cloudflare Logpush logo", "size": "216x216", "type": "image/svg+xml" } ], "conditions": { "kibana": { "version": "^8.17.1 || ^9.0.0" } }, "owner": { "type": "elastic", "github": "elastic/security-service-integrations" }, "categories": [ "security", "network", "cdn_security" ], "signature_path": "/epr/cloudflare_logpush/cloudflare_logpush-1.44.0.zip.sig", "format_version": "3.0.2", "readme": "/package/cloudflare_logpush/1.44.0/docs/README.md", "license": "basic", "screenshots": [ { "src": "/img/cloudflare_logpush-overview1.png", "path": "/package/cloudflare_logpush/1.44.0/img/cloudflare_logpush-overview1.png", "title": "Cloudflare Logpush - Zero Trust Overview", "size": "1847x950", "type": "image/png" }, { "src": "/img/cloudflare_logpush-audit1.png", "path": "/package/cloudflare_logpush/1.44.0/img/cloudflare_logpush-audit1.png", "title": "Cloudflare Logpush - Audit", "size": "1850x948", "type": "image/png" }, { "src": "/img/cloudflare_logpush-gateway_http1.png", "path": "/package/cloudflare_logpush/1.44.0/img/cloudflare_logpush-gateway_http1.png", "title": "Cloudflare - Gateway HTTP", "size": "1850x948", "type": "image/png" }, { "src": "/img/cloudflare_logpush-network_sessions1.png", "path": "/package/cloudflare_logpush/1.44.0/img/cloudflare_logpush-network_sessions1.png", "title": "Cloudflare - Network Sessions", "size": "1850x948", "type": "image/png" }, { "src": "/img/cloudflare_logpush-workers_trace1.png", "path": "/package/cloudflare_logpush/1.44.0/img/cloudflare_logpush-workers_trace1.png", "title": "Cloudflare - Workers Trace", "size": "1850x948", "type": "image/png" } ], "assets": [ "/package/cloudflare_logpush/1.44.0/LICENSE.txt", "/package/cloudflare_logpush/1.44.0/changelog.yml", "/package/cloudflare_logpush/1.44.0/manifest.yml", "/package/cloudflare_logpush/1.44.0/validation.yml", "/package/cloudflare_logpush/1.44.0/docs/README.md", "/package/cloudflare_logpush/1.44.0/img/cloudflare-logo.svg", "/package/cloudflare_logpush/1.44.0/img/cloudflare_logpush-audit1.png", "/package/cloudflare_logpush/1.44.0/img/cloudflare_logpush-gateway_http1.png", "/package/cloudflare_logpush/1.44.0/img/cloudflare_logpush-network_sessions1.png", "/package/cloudflare_logpush/1.44.0/img/cloudflare_logpush-overview1.png", "/package/cloudflare_logpush/1.44.0/img/cloudflare_logpush-workers_trace1.png", "/package/cloudflare_logpush/1.44.0/kibana/tags.yml", "/package/cloudflare_logpush/1.44.0/data_stream/access_request/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/access_request/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/audit/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/audit/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/casb/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/casb/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/device_posture/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/device_posture/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/dlp_forensic_copies/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dlp_forensic_copies/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/dns/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dns/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/dns_firewall/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dns_firewall/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/email_security_alerts/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/email_security_alerts/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/firewall_event/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/firewall_event/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_dns/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_dns/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_http/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_http/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_network/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_network/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/http_request/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/http_request/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/magic_ids/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/magic_ids/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/nel_report/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/nel_report/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/network_analytics/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/network_analytics/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/network_session/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/network_session/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/page_shield_events/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/page_shield_events/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/sinkhole_http/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/sinkhole_http/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/spectrum_event/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/spectrum_event/sample_event.json", "/package/cloudflare_logpush/1.44.0/data_stream/workers_trace/manifest.yml", "/package/cloudflare_logpush/1.44.0/data_stream/workers_trace/sample_event.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-09ef2620-58a5-11ee-99a4-d186f5240e2e.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-0f1c8080-04b2-11ee-8373-5fabaa30ca64.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-11e340a0-04bd-11ee-8373-5fabaa30ca64.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-3da7bd20-dc45-11ec-b76d-adcfe05cc1fe.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-5b1248c0-044a-11ee-8373-5fabaa30ca64.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-5de148d0-04ae-11ee-8373-5fabaa30ca64.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-67684710-0151-11ee-a051-4174215e3eb2.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-6f14e840-0450-11ee-8373-5fabaa30ca64.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-78731300-047b-11ee-8373-5fabaa30ca64.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-87f6ad60-dc44-11ec-b76d-adcfe05cc1fe.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-8ae621f0-04ba-11ee-8373-5fabaa30ca64.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-940b5bb0-015e-11ee-a051-4174215e3eb2.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-9c472060-58c6-11ee-99a4-d186f5240e2e.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-a32a0690-dc44-11ec-b76d-adcfe05cc1fe.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-a32dd850-2ad7-11ee-87da-672c93538971.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-bb426420-dc44-11ec-b76d-adcfe05cc1fe.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-cc375d30-dc44-11ec-b76d-adcfe05cc1fe.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-da55ddb0-dc44-11ec-b76d-adcfe05cc1fe.json", "/package/cloudflare_logpush/1.44.0/kibana/dashboard/cloudflare_logpush-fe4ebba0-58bd-11ee-99a4-d186f5240e2e.json", "/package/cloudflare_logpush/1.44.0/kibana/search/cloudflare_logpush-65cf2c20-0886-11ee-9009-9d20eb6e9358.json", "/package/cloudflare_logpush/1.44.0/kibana/search/cloudflare_logpush-79eaa3e0-58a6-11ee-99a4-d186f5240e2e.json", "/package/cloudflare_logpush/1.44.0/kibana/search/cloudflare_logpush-9d429e20-0887-11ee-9009-9d20eb6e9358.json", "/package/cloudflare_logpush/1.44.0/kibana/search/cloudflare_logpush-a58b3a80-e257-11ec-b57d-b9b9d5221e36.json", "/package/cloudflare_logpush/1.44.0/kibana/search/cloudflare_logpush-c1961a90-2ad3-11ee-87da-672c93538971.json", "/package/cloudflare_logpush/1.44.0/kibana/search/cloudflare_logpush-dc01afe0-e24d-11ec-b57d-b9b9d5221e36.json", "/package/cloudflare_logpush/1.44.0/data_stream/access_request/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/access_request/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/access_request/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/access_request/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/audit/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/audit/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/audit/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/audit/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/casb/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/casb/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/casb/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/casb/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/device_posture/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/device_posture/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/device_posture/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/device_posture/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dlp_forensic_copies/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dlp_forensic_copies/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dlp_forensic_copies/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dlp_forensic_copies/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dns/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dns/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dns/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dns/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dns_firewall/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dns_firewall/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dns_firewall/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dns_firewall/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/email_security_alerts/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/email_security_alerts/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/email_security_alerts/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/email_security_alerts/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/firewall_event/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/firewall_event/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/firewall_event/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/firewall_event/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_dns/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_dns/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_dns/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_dns/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_http/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_http/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_http/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_http/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_network/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_network/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_network/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_network/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/http_request/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/http_request/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/http_request/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/http_request/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/magic_ids/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/magic_ids/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/magic_ids/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/magic_ids/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/nel_report/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/nel_report/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/nel_report/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/nel_report/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/network_analytics/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/network_analytics/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/network_analytics/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/network_analytics/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/network_session/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/network_session/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/network_session/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/network_session/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/page_shield_events/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/page_shield_events/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/page_shield_events/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/page_shield_events/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/sinkhole_http/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/sinkhole_http/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/sinkhole_http/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/sinkhole_http/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/spectrum_event/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/spectrum_event/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/spectrum_event/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/spectrum_event/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/workers_trace/fields/agent.yml", "/package/cloudflare_logpush/1.44.0/data_stream/workers_trace/fields/base-fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/workers_trace/fields/beats.yml", "/package/cloudflare_logpush/1.44.0/data_stream/workers_trace/fields/fields.yml", "/package/cloudflare_logpush/1.44.0/data_stream/access_request/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/access_request/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/access_request/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/access_request/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/access_request/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/audit/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/audit/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/audit/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/audit/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/audit/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/casb/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/casb/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/casb/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/casb/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/casb/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/device_posture/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/device_posture/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/device_posture/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/device_posture/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/device_posture/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dlp_forensic_copies/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/dlp_forensic_copies/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/dlp_forensic_copies/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/dlp_forensic_copies/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dns/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/dns/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/dns/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/dns/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/dns/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/dns_firewall/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/dns_firewall/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/dns_firewall/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/dns_firewall/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/dns_firewall/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/email_security_alerts/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/email_security_alerts/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/email_security_alerts/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/email_security_alerts/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/email_security_alerts/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/firewall_event/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/firewall_event/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/firewall_event/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/firewall_event/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_dns/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_dns/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_dns/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_dns/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_http/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_http/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_http/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_http/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_network/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_network/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_network/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_network/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/http_request/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/http_request/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/http_request/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/http_request/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/magic_ids/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/magic_ids/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/magic_ids/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/magic_ids/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/magic_ids/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/nel_report/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/nel_report/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/nel_report/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/nel_report/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/nel_report/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/network_analytics/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/network_analytics/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/network_analytics/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/network_analytics/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/network_analytics/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/network_session/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/network_session/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/network_session/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/network_session/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/network_session/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/page_shield_events/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/page_shield_events/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/page_shield_events/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/page_shield_events/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/page_shield_events/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/sinkhole_http/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/sinkhole_http/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/sinkhole_http/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/sinkhole_http/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/sinkhole_http/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/spectrum_event/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/spectrum_event/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/spectrum_event/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/spectrum_event/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/spectrum_event/elasticsearch/ingest_pipeline/default.yml", "/package/cloudflare_logpush/1.44.0/data_stream/workers_trace/agent/stream/abs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/workers_trace/agent/stream/aws-s3.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/workers_trace/agent/stream/gcs.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/workers_trace/agent/stream/http_endpoint.yml.hbs", "/package/cloudflare_logpush/1.44.0/data_stream/workers_trace/elasticsearch/ingest_pipeline/default.yml" ], "policy_templates": [ { "name": "cloudflare", "title": "Cloudflare Logpush logs", "description": "Collect logs from Cloudflare.", "inputs": [ { "type": "http_endpoint", "vars": [ { "name": "listen_address", "type": "text", "title": "Listen Address", "description": "The bind address to listen for http endpoint connections. Set to `0.0.0.0` to bind to all available interfaces.", "multi": false, "required": true, "show_user": true, "default": "localhost" }, { "name": "secret_header", "type": "text", "title": "Secret Header", "description": "The header to check for a specific value specified by `secret.value`.", "multi": false, "required": false, "show_user": false }, { "name": "secret_value", "type": "password", "title": "Secret Value", "description": "The secret stored in the header name specified by `secret.header`.", "multi": false, "required": false, "show_user": false }, { "name": "ssl", "type": "yaml", "title": "SSL Configuration", "description": "SSL configuration options. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-ssl.html#ssl-common-config) for details.", "multi": false, "required": false, "show_user": false, "default": "#certificate_authorities:\n# - |\n# -----BEGIN CERTIFICATE-----\n# MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF\n# ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2\n# MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB\n# BQADggEPADCCAQoCggEBANce58Y/JykI58iyOXpxGfw0/gMvF0hUQAcUrSMxEO6n\n# fZRA49b4OV4SwWmA3395uL2eB2NB8y8qdQ9muXUdPBWE4l9rMZ6gmfu90N5B5uEl\n# 94NcfBfYOKi1fJQ9i7WKhTjlRkMCgBkWPkUokvBZFRt8RtF7zI77BSEorHGQCk9t\n# /D7BS0GJyfVEhftbWcFEAG3VRcoMhF7kUzYwp+qESoriFRYLeDWv68ZOvG7eoWnP\n# PsvZStEVEimjvK5NSESEQa9xWyJOmlOKXhkdymtcUd/nXnx6UTCFgnkgzSdTWV41\n# CI6B6aJ9svCTI2QuoIq2HxX/ix7OvW1huVmcyHVxyUECAwEAAaNTMFEwHQYDVR0O\n# BBYEFPwN1OceFGm9v6ux8G+DZ3TUDYxqMB8GA1UdIwQYMBaAFPwN1OceFGm9v6ux\n# 8G+DZ3TUDYxqMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAG5D\n# 874A4YI7YUwOVsVAdbWtgp1d0zKcPRR+r2OdSbTAV5/gcS3jgBJ3i1BN34JuDVFw\n# 3DeJSYT3nxy2Y56lLnxDeF8CUTUtVQx3CuGkRg1ouGAHpO/6OqOhwLLorEmxi7tA\n# H2O8mtT0poX5AnOAhzVy7QW0D/k4WaoLyckM5hUa6RtvgvLxOwA0U+VGurCDoctu\n# 8F4QOgTAWyh8EZIwaKCliFRSynDpv3JTUwtfZkxo6K6nce1RhCWFAsMvDZL8Dgc0\n# yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk\n# sxSmbIUfc2SGJGCJD4I=\n# -----END CERTIFICATE-----\n" } ], "title": "Collect Cloudflare Logpush logs via HTTP Endpoint", "description": "Collecting Logpush logs from Cloudflare via HTTP Endpoint." }, { "type": "aws-s3", "vars": [ { "name": "collect_s3_logs", "type": "bool", "title": "Collect logs via S3 Bucket", "description": "To Collect logs via S3 bucket enable the toggle switch. By default, it will collect logs via SQS Queue.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "bucket_arn", "type": "text", "title": "[S3] Bucket ARN", "description": "ARN of the AWS S3 bucket that will be polled for list operation. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.", "multi": false, "required": false, "show_user": true }, { "name": "access_point_arn", "type": "text", "title": "[S3] Access Point ARN", "description": "ARN of the AWS S3 Access Point that will be polled for list operation. Mandatory if the \"Collect logs via S3 Bucket\" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.", "multi": false, "required": false, "show_user": true }, { "name": "cloudflare_r2", "type": "text", "title": "[Global][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. This is a global setting which can be overriden by specific local bucket names for each data stream if required.\nUsing non-AWS S3 compatible buckets requires the use of Access Key ID and Secret Access Key for authentication. To specify the non-AWS S3 bucket name, use the non_aws_bucket_name config and the endpoint must be set to replace the default API endpoint.", "multi": false, "required": false, "show_user": true }, { "name": "access_key_id", "type": "password", "title": "Access Key ID", "description": "First part of access key.", "multi": false, "required": false, "show_user": true }, { "name": "secret_access_key", "type": "password", "title": "Secret Access Key", "description": "Second part of access key.", "multi": false, "required": false, "show_user": true }, { "name": "session_token", "type": "password", "title": "Session Token", "description": "Required when using temporary security credentials.", "multi": false, "required": false, "show_user": true }, { "name": "shared_credential_file", "type": "text", "title": "Shared Credential File", "description": "Directory of the shared credentials file.", "multi": false, "required": false, "show_user": false }, { "name": "credential_profile_name", "type": "text", "title": "Credential Profile Name", "description": "Profile name in shared credentials file.", "multi": false, "required": false, "show_user": false }, { "name": "role_arn", "type": "text", "title": "Role ARN", "description": "AWS IAM Role to assume.", "multi": false, "required": false, "show_user": false }, { "name": "endpoint", "type": "text", "title": "Endpoint", "description": "URL of the entry point for an AWS web service.", "multi": false, "required": false, "show_user": false, "default": "" }, { "name": "region", "type": "text", "title": "Region", "description": "The name of the AWS region of the end point. Required when using a non-AWS S3-compatible bucket (e.g. Cloudflare R2).", "multi": false, "required": false, "show_user": false, "default": "" }, { "name": "default_region", "type": "text", "title": "Default AWS Region", "description": "Default region to use prior to connecting to region specific services/endpoints if no AWS region is set from environment variable, credentials or instance profile. If none of the above are set and no default region is set as well, `us-east-1` is used. A region, either from environment variable, credentials or instance profile or from this default region setting, needs to be set when using regions in non-regular AWS environments such as AWS China or US Government Isolated.", "multi": false, "required": false, "show_user": false, "default": "" }, { "name": "fips_enabled", "type": "bool", "title": "Enable S3 FIPS", "description": "Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.", "multi": false, "required": false, "show_user": false, "default": false }, { "name": "proxy_url", "type": "text", "title": "Proxy URL", "description": "URL to proxy connections in the form of http[s]://:@:. Please ensure your username and password are in URL encoded format.", "multi": false, "required": false, "show_user": false }, { "name": "ssl", "type": "yaml", "title": "SSL Configuration", "description": "SSL configuration options. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-ssl.html#ssl-common-config) for details.", "multi": false, "required": false, "show_user": false, "default": "#certificate_authorities:\n# - |\n# -----BEGIN CERTIFICATE-----\n# MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF\n# ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2\n# MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB\n# BQADggEPADCCAQoCggEBANce58Y/JykI58iyOXpxGfw0/gMvF0hUQAcUrSMxEO6n\n# fZRA49b4OV4SwWmA3395uL2eB2NB8y8qdQ9muXUdPBWE4l9rMZ6gmfu90N5B5uEl\n# 94NcfBfYOKi1fJQ9i7WKhTjlRkMCgBkWPkUokvBZFRt8RtF7zI77BSEorHGQCk9t\n# /D7BS0GJyfVEhftbWcFEAG3VRcoMhF7kUzYwp+qESoriFRYLeDWv68ZOvG7eoWnP\n# PsvZStEVEimjvK5NSESEQa9xWyJOmlOKXhkdymtcUd/nXnx6UTCFgnkgzSdTWV41\n# CI6B6aJ9svCTI2QuoIq2HxX/ix7OvW1huVmcyHVxyUECAwEAAaNTMFEwHQYDVR0O\n# BBYEFPwN1OceFGm9v6ux8G+DZ3TUDYxqMB8GA1UdIwQYMBaAFPwN1OceFGm9v6ux\n# 8G+DZ3TUDYxqMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAG5D\n# 874A4YI7YUwOVsVAdbWtgp1d0zKcPRR+r2OdSbTAV5/gcS3jgBJ3i1BN34JuDVFw\n# 3DeJSYT3nxy2Y56lLnxDeF8CUTUtVQx3CuGkRg1ouGAHpO/6OqOhwLLorEmxi7tA\n# H2O8mtT0poX5AnOAhzVy7QW0D/k4WaoLyckM5hUa6RtvgvLxOwA0U+VGurCDoctu\n# 8F4QOgTAWyh8EZIwaKCliFRSynDpv3JTUwtfZkxo6K6nce1RhCWFAsMvDZL8Dgc0\n# yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk\n# sxSmbIUfc2SGJGCJD4I=\n# -----END CERTIFICATE-----\n" } ], "title": "Collect Cloudflare Logpush logs via AWS S3, AWS SQS, or S3-Compatible Buckets.", "description": "Collecting Logpush logs from Cloudflare via AWS S3, AWS SQS, or S3-Compatible Buckets." }, { "type": "gcs", "vars": [ { "name": "project_id", "type": "text", "title": "Project Id", "description": "It is a required parameter to collect logs via GCS.", "multi": false, "required": true, "show_user": true, "default": "my-project-id" }, { "name": "service_account_key", "type": "password", "title": "JSON Credentials key", "description": "The service account key text to authenticate to the data store.", "multi": false, "required": false, "show_user": true }, { "name": "service_account_file", "type": "text", "title": "JSON Credentials file path", "description": "Path to a file containing the service account key text to authenticate to the data store.", "multi": false, "required": false, "show_user": false } ], "title": "Collect Cloudflare Logpush logs via Google Cloud Storage", "description": "Collecting Logpush logs from Cloudflare via Google Cloud Storage." }, { "type": "azure-blob-storage", "vars": [ { "name": "account_name", "type": "text", "title": "Account Name", "description": "This attribute is required for various internal operations with respect to authentication, creating service clients and blob clients which are used internally for various processing purposes.\n", "multi": false, "required": true, "show_user": true }, { "name": "storage_url", "type": "text", "title": "Storage URL", "description": "Use this attribute to specify a custom storage URL if required. By default it points to azure cloud storage. Only use this if there is a specific need to connect to a different environment where blob storage is available.\nURL format : {{protocol}}://{{account_name}}.{{storage_uri}}.\n", "multi": false, "required": false, "show_user": false }, { "name": "oauth2", "type": "bool", "title": "Collect logs using OAuth2 authentication", "description": "To collect logs using OAuth2 authentication enable the toggle switch. By default, it will collect logs using service account key or URI.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "client_id", "type": "text", "title": "Client ID (OAuth2)", "description": "Client ID of Azure Account. This is required if 'Collect logs using OAuth2 authentication' is enabled.", "multi": false, "required": false, "show_user": true }, { "name": "client_secret", "type": "password", "title": "Client Secret (OAuth2)", "description": "Client Secret of Azure Account. This is required if 'Collect logs using OAuth2 authentication' is enabled.", "multi": false, "required": false, "show_user": true }, { "name": "tenant_id", "type": "text", "title": "Tenant ID (OAuth2)", "description": "Tenant ID of Azure Account. This is required if 'Collect logs using OAuth2 authentication' is enabled.", "multi": false, "required": false, "show_user": true }, { "name": "service_account_key", "type": "password", "title": "Service Account Key", "description": "This attribute contains the access key, found under the Access keys section on Azure Cloud, under the respective storage account. A single storage account can contain multiple containers, and they will all use this common access key.\n", "multi": false, "required": false, "show_user": true }, { "name": "service_account_uri", "type": "text", "title": "Service Account URI", "description": "This attribute contains the connection string, found under the Access keys section on Azure Cloud, under the respective storage account. A single storage account can contain multiple containers, and they will all use this common connection string.\n", "multi": false, "required": false, "show_user": false } ], "title": "Collect Cloudflare Logpush logs via Azure Blob Storage", "description": "Collecting Logpush logs from Cloudflare via Azure Blob Storage." } ], "multiple": true } ], "data_streams": [ { "type": "logs", "dataset": "cloudflare_logpush.access_request", "title": "Collect Access Request logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/access_request.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/access_request" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-access_request" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Access Request logs", "description": "Collect Access Request logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_access_request", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from.\nThis is only required if you want to collect logs via AWS SQS.\nThis is a Access Request data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "access_request" }, { "name": "cloudflare_r2_access_request", "type": "text", "title": "[Access Request][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "start_timestamp", "type": "text", "title": "[S3] Start Timestamp", "description": "If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, \"2020-10-10T10:30:00Z\" (UTC) or \"2020-10-10T10:30:00Z+02:30\" (with zone offset).", "multi": false, "required": false, "show_user": false }, { "name": "ignore_older", "type": "text", "title": "[S3] Ignore Older Timespan", "description": "If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.", "multi": false, "required": false, "show_user": false }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'access_request/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-access_request" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Access Request logs", "description": "Collect Access Request logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: access_request_logs\n- name: access_request_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: access_request_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-access_request" ] } ], "template_path": "gcs.yml.hbs", "title": "Access Request logs", "description": "Collect Access Request logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-access_request" ] } ], "template_path": "abs.yml.hbs", "title": "Access Request logs", "description": "Collect Access Request logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "access_request" }, { "type": "logs", "dataset": "cloudflare_logpush.audit", "title": "Collect Audit logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/audit.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/audit" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-audit" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Audit logs", "description": "Collect Audit logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_audit", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from. \nThis is only required if you want to collect logs via AWS SQS.\nThis is an audit data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "audit_logs" }, { "name": "cloudflare_r2_audit", "type": "text", "title": "[Audit][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "start_timestamp", "type": "text", "title": "[S3] Start Timestamp", "description": "If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, \"2020-10-10T10:30:00Z\" (UTC) or \"2020-10-10T10:30:00Z+02:30\" (with zone offset).", "multi": false, "required": false, "show_user": false }, { "name": "ignore_older", "type": "text", "title": "[S3] Ignore Older Timespan", "description": "If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.", "multi": false, "required": false, "show_user": false }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'audit_logs/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-audit" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Audit logs", "description": "Collect Audit logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: audit_logs\n- name: audit_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: audit_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-audit" ] } ], "template_path": "gcs.yml.hbs", "title": "Audit logs", "description": "Collect Audit logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-audit" ] } ], "template_path": "abs.yml.hbs", "title": "Audit logs", "description": "Collect Audit logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "audit" }, { "type": "logs", "dataset": "cloudflare_logpush.casb", "title": "Collect CASB Findings logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/casb.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/casb" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-casb" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "CASB Findings logs", "description": "Collect CASB Findings logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_casb", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from.\nThis is only required if you want to collect logs via AWS SQS.\nThis is a CASB Findings data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "casb" }, { "name": "cloudflare_r2_casb", "type": "text", "title": "[CASB Findings][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "start_timestamp", "type": "text", "title": "[S3] Start Timestamp", "description": "If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, \"2020-10-10T10:30:00Z\" (UTC) or \"2020-10-10T10:30:00Z+02:30\" (with zone offset).", "multi": false, "required": false, "show_user": false }, { "name": "ignore_older", "type": "text", "title": "[S3] Ignore Older Timespan", "description": "If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.", "multi": false, "required": false, "show_user": false }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'casb/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-casb" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "CASB Findings logs", "description": "Collect CASB Findings logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: casb_logs\n- name: casb_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: casb_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-casb" ] } ], "template_path": "gcs.yml.hbs", "title": "CASB Findings logs", "description": "Collect CASB Findings logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-casb" ] } ], "template_path": "abs.yml.hbs", "title": "CASB Findings logs", "description": "Collect CASB Findings logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "casb" }, { "type": "logs", "dataset": "cloudflare_logpush.device_posture", "title": "Collect Device Posture Results logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/device_posture.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/device_posture" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-device_posture" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Device Posture Results logs", "description": "Collect Device Posture Results logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_device_posture", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from.\nThis is only required if you want to collect logs via AWS SQS.\nThis is a Device Posture Results data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "device_posture" }, { "name": "cloudflare_r2_device_posture", "type": "text", "title": "[Device Posture Results][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'device_posture/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-device_posture" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Device Posture Results logs", "description": "Collect Device Posture Results logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: device_posture_logs\n- name: device_posture_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: device_posture_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-device_posture" ] } ], "template_path": "gcs.yml.hbs", "title": "Device Posture Results logs", "description": "Collect Device Posture Results logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-device_posture" ] } ], "template_path": "abs.yml.hbs", "title": "Device Posture Results logs logs", "description": "Collect Device Posture Results logs logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "device_posture" }, { "type": "logs", "dataset": "cloudflare_logpush.dlp_forensic_copies", "title": "Collect DLP Forensic Copies logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/dlp_forensic_copies.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/dlp_forensic_copies" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-dlp_forensic_copies" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "DLP Forensic Copies logs", "description": "Collect DLP Forensic Copies logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_dlp_forensic_copies", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from. \nThis is only required if you want to collect logs via AWS SQS.\nThis is a DLP Forensic Copies data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "dlp_forensic_copies" }, { "name": "cloudflare_r2_dlp_forensic_copies", "type": "text", "title": "[DLP Forensic Copies][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3] Number of Workers", "description": "Number of workers that will process the S3 objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'dlp_forensic_copies/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-dlp_forensic_copies" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "DLP Forensic Copies logs", "description": "Collect DLP Forensic Copies logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: dlp_forensic_copies_logs\n- name: dlp_forensic_copies_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: dlp_forensic_copies_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-dlp_forensic_copies" ] } ], "template_path": "gcs.yml.hbs", "title": "DLP Forensic Copies logs", "description": "Collect DLP Forensic Copies logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-dlp_forensic_copies" ] } ], "template_path": "abs.yml.hbs", "title": "DLP Forensic Copies logs", "description": "Collect DLP Forensic Copies logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "dlp_forensic_copies" }, { "type": "logs", "dataset": "cloudflare_logpush.dns", "title": "Collect DNS logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/dns.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/dns" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-dns" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "DNS logs", "description": "Collect DNS logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_dns", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from. \nThis is only required if you want to collect logs via AWS SQS.\nThis is a dns data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "dns" }, { "name": "cloudflare_r2_dns", "type": "text", "title": "[DNS][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "start_timestamp", "type": "text", "title": "[S3] Start Timestamp", "description": "If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, \"2020-10-10T10:30:00Z\" (UTC) or \"2020-10-10T10:30:00Z+02:30\" (with zone offset).", "multi": false, "required": false, "show_user": false }, { "name": "ignore_older", "type": "text", "title": "[S3] Ignore Older Timespan", "description": "If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.", "multi": false, "required": false, "show_user": false }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'dns/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-dns" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "DNS logs", "description": "Collect DNS logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: dns_logs\n- name: dns_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: dns_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-dns" ] } ], "template_path": "gcs.yml.hbs", "title": "DNS logs", "description": "Collect DNS logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-dns" ] } ], "template_path": "abs.yml.hbs", "title": "DNS logs", "description": "Collect DNS logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "dns" }, { "type": "logs", "dataset": "cloudflare_logpush.dns_firewall", "title": "Collect DNS Firewall logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/dns_firewall.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/dns_firewall" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-dns_firewall" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "DNS Firewall logs", "description": "Collect DNS Firewall logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_dns_firewall", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from.\nThis is only required if you want to collect logs via AWS SQS.\nThis is a DNS Firewall data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "dns_firewall" }, { "name": "cloudflare_r2_dns_firewall", "type": "text", "title": "[DNS Firewall][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'dns_firewall/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-dns_firewall" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "DNS Firewall logs", "description": "Collect DNS Firewall logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: dns_firewall_logs\n- name: dns_firewall_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: dns_firewall_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-dns_firewall" ] } ], "template_path": "gcs.yml.hbs", "title": "DNS Firewall logs", "description": "Collect DNS Firewall logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-dns_firewall" ] } ], "template_path": "abs.yml.hbs", "title": "DNS Firewall logs", "description": "Collect DNS Firewall logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "dns_firewall" }, { "type": "logs", "dataset": "cloudflare_logpush.email_security_alerts", "title": "Collect Email Security Alert logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/email_security_alerts.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/email_security_alerts" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-email_security_alerts" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Email Security Alert logs", "description": "Collect Email Security Alert logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_email_security_alerts", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from. \nThis is only required if you want to collect logs via AWS SQS.\nThis is a Email Security Alert data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "email_security_alerts" }, { "name": "cloudflare_r2_email_security_alerts", "type": "text", "title": "[Email Security Alert][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3] Number of Workers", "description": "Number of workers that will process the S3 objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'email_security_alerts/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-email_security_alerts" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Email Security Alert logs", "description": "Collect Email Security Alert logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: email_security_alerts_logs\n- name: email_security_alerts_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: email_security_alerts_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-email_security_alerts" ] } ], "template_path": "gcs.yml.hbs", "title": "Email Security Alert logs", "description": "Collect Email Security Alert logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-email_security_alerts" ] } ], "template_path": "abs.yml.hbs", "title": "Email Security Alert logs", "description": "Collect Email Security Alert logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "email_security_alerts" }, { "type": "logs", "dataset": "cloudflare_logpush.firewall_event", "title": "Collect Firewall Event logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/firewall_event.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/firewall_event" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-firewall_event" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Firewall Event logs", "description": "Collect Firewall Event logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_firewall_event", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from. \nThis is only required if you want to collect logs via AWS SQS.\nThis is a firewall event data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "firewall_event" }, { "name": "cloudflare_r2_firewall_event", "type": "text", "title": "[Firewall Event][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'firewall_event/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-firewall_event" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Firewall Event logs", "description": "Collect Firewall Event logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: firewall_event_logs\n- name: firewall_event_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: firewall_event_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-firewall_event" ] } ], "template_path": "gcs.yml.hbs", "title": "Firewall Event logs", "description": "Collect Firewall Event logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-firewall_event" ] } ], "template_path": "abs.yml.hbs", "title": "Firewall Event logs", "description": "Collect Firewall Event logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "firewall_event" }, { "type": "logs", "dataset": "cloudflare_logpush.gateway_dns", "title": "Collect Gateway DNS logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/gateway_dns.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/gateway_dns" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-gateway_dns" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Gateway DNS logs", "description": "Collect Gateway DNS logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_gateway_dns", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from.\nThis is only required if you want to collect logs via AWS SQS.\nThis is a Gateway DNS data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "gateway_dns" }, { "name": "cloudflare_r2_gateway_dns", "type": "text", "title": "[Gateway DNS][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'gateway_dns/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-gateway_dns" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Gateway DNS logs", "description": "Collect Gateway DNS logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: gateway_dns_logs\n- name: gateway_dns_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: gateway_dns_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-gateway_dns" ] } ], "template_path": "gcs.yml.hbs", "title": "Gateway DNS logs", "description": "Collect Gateway DNS logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-gateway_dns" ] } ], "template_path": "abs.yml.hbs", "title": "Gateway DNS logs", "description": "Collect Gateway DNS logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "gateway_dns" }, { "type": "logs", "dataset": "cloudflare_logpush.gateway_http", "title": "Collect Gateway HTTP logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/gateway_http.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/gateway_http" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-gateway_http" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Gateway HTTP logs", "description": "Collect Gateway HTTP logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_gateway_http", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from.\nThis is only required if you want to collect logs via AWS SQS.\nThis is a Gateway HTTP data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "gateway_http" }, { "name": "cloudflare_r2_gateway_http", "type": "text", "title": "[Gateway HTTP][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'gateway_http/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-gateway_http" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Gateway HTTP logs", "description": "Collect Gateway HTTP logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: gateway_http_logs\n- name: gateway_http_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: gateway_http_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-gateway_http" ] } ], "template_path": "gcs.yml.hbs", "title": "Gateway HTTP logs", "description": "Collect Gateway HTTP logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-gateway_http" ] } ], "template_path": "abs.yml.hbs", "title": "Gateway HTTP logs", "description": "Collect Gateway HTTP logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "gateway_http" }, { "type": "logs", "dataset": "cloudflare_logpush.gateway_network", "title": "Collect Gateway Network logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/gateway_network.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/gateway_network" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-gateway_network" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Gateway Network logs", "description": "Collect Gateway Network logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_gateway_network", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from.\nThis is only required if you want to collect logs via AWS SQS.\nThis is a Gateway Network data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "gateway_network" }, { "name": "cloudflare_r2_gateway_network", "type": "text", "title": "[Gateway Network][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'gateway_network/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-gateway_network" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Gateway Network logs", "description": "Collect Gateway Network logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: gateway_network_logs\n- name: gateway_network_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: gateway_network_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-gateway_network" ] } ], "template_path": "gcs.yml.hbs", "title": "Gateway Network logs", "description": "Collect Gateway Network logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-gateway_network" ] } ], "template_path": "abs.yml.hbs", "title": "Gateway Network logs", "description": "Collect Gateway Network logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "gateway_network" }, { "type": "logs", "dataset": "cloudflare_logpush.http_request", "title": "Collect HTTP Request logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/http_request.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/http_request" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-http_request" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "HTTP Request logs", "description": "Collect HTTP Request logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_http_request", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from. \nThis is only required if you want to collect logs via AWS SQS.\nThis is a http request data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "http_request" }, { "name": "cloudflare_r2_http_request", "type": "text", "title": "[HTTP Request][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'http_request/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-http_request" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "HTTP Request logs", "description": "Collect HTTP Request logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: http_request_logs\n- name: http_request_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: http_request_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-http_request" ] } ], "template_path": "gcs.yml.hbs", "title": "HTTP Request logs", "description": "Collect HTTP Request logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-http_request" ] } ], "template_path": "abs.yml.hbs", "title": "HTTP request logs", "description": "Collect HTTP request logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "http_request" }, { "type": "logs", "dataset": "cloudflare_logpush.magic_ids", "title": "Collect Magic IDS logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/magic_ids.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/magic_ids" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-magic_ids" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Magic IDS logs", "description": "Collect Magic IDS logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_magic_ids", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from.\nThis is only required if you want to collect logs via AWS SQS.\nThis is a Magic IDS data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "magic_ids" }, { "name": "cloudflare_r2_magic_ids", "type": "text", "title": "[Magic IDS][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "start_timestamp", "type": "text", "title": "[S3] Start Timestamp", "description": "If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, \"2020-10-10T10:30:00Z\" (UTC) or \"2020-10-10T10:30:00Z+02:30\" (with zone offset).", "multi": false, "required": false, "show_user": false }, { "name": "ignore_older", "type": "text", "title": "[S3] Ignore Older Timespan", "description": "If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.", "multi": false, "required": false, "show_user": false }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'magic_ids/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-magic_ids" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Magic IDS logs", "description": "Collect Magic IDS logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: magic_ids_logs\n- name: magic_ids_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: magic_ids_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-magic_ids" ] } ], "template_path": "gcs.yml.hbs", "title": "Magic IDS logs", "description": "Collect Magic IDS logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-magic_ids" ] } ], "template_path": "abs.yml.hbs", "title": "Magic IDS logs", "description": "Collect Magic IDS logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "magic_ids" }, { "type": "logs", "dataset": "cloudflare_logpush.nel_report", "title": "Collect NEL Report logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/nel_report.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/nel_report" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-nel_report" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "NEL Report logs", "description": "Collect NEL Report logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_nel_report", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from. \nThis is only required if you want to collect logs via AWS SQS.\nThis is a nel report data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "nel_report" }, { "name": "cloudflare_r2_nel_report", "type": "text", "title": "[NEL Report][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "start_timestamp", "type": "text", "title": "[S3] Start Timestamp", "description": "If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, \"2020-10-10T10:30:00Z\" (UTC) or \"2020-10-10T10:30:00Z+02:30\" (with zone offset).", "multi": false, "required": false, "show_user": false }, { "name": "ignore_older", "type": "text", "title": "[S3] Ignore Older Timespan", "description": "If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.", "multi": false, "required": false, "show_user": false }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'nel_report/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-nel_report" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "NEL Report logs", "description": "Collect NEL Report logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: nel_report_logs\n- name: nel_report_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: nel_report_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-nel_report" ] } ], "template_path": "gcs.yml.hbs", "title": "NEL Report logs", "description": "Collect NEL Report logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-nel_report" ] } ], "template_path": "abs.yml.hbs", "title": "NEL Report logs", "description": "Collect NEL Report logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "nel_report" }, { "type": "logs", "dataset": "cloudflare_logpush.network_analytics", "title": "Collect Network Analytics logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/network_analytics.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/network_analytics" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-network_analytics" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Network Analytics logs", "description": "Collect Network Analytics logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_network_analytics", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from. \nThis is only required if you want to collect logs via AWS SQS.\nThis is a network analytics data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "network_analytics_logs" }, { "name": "cloudflare_r2_network_analytics", "type": "text", "title": "[Network Analytics][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "start_timestamp", "type": "text", "title": "[S3] Start Timestamp", "description": "If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, \"2020-10-10T10:30:00Z\" (UTC) or \"2020-10-10T10:30:00Z+02:30\" (with zone offset).", "multi": false, "required": false, "show_user": false }, { "name": "ignore_older", "type": "text", "title": "[S3] Ignore Older Timespan", "description": "If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.", "multi": false, "required": false, "show_user": false }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'network_analytics_logs/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-network_analytics" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Network Analytics logs", "description": "Collect Network Analytics logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: network_analytics_logs\n- name: network_analytics_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: network_analytics_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-network_analytics" ] } ], "template_path": "gcs.yml.hbs", "title": "Network Analytics logs", "description": "Collect Network Analytics logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-network_analytics" ] } ], "template_path": "abs.yml.hbs", "title": "Network Analytics logs", "description": "Collect Network Analytics logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "network_analytics" }, { "type": "logs", "dataset": "cloudflare_logpush.network_session", "title": "Collect Zero Trust Network Session logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/network_session.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/network_session" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-network_session" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Zero Trust Network Session logs", "description": "Collect Zero Trust Network Session logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_network_session", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from.\nThis is only required if you want to collect logs via AWS SQS.\nThis is a Zero Trust Network Session data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "network_session" }, { "name": "cloudflare_r2_network_session", "type": "text", "title": "[Zero Trust Network Session][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "start_timestamp", "type": "text", "title": "[S3] Start Timestamp", "description": "If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, \"2020-10-10T10:30:00Z\" (UTC) or \"2020-10-10T10:30:00Z+02:30\" (with zone offset).", "multi": false, "required": false, "show_user": false }, { "name": "ignore_older", "type": "text", "title": "[S3] Ignore Older Timespan", "description": "If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.", "multi": false, "required": false, "show_user": false }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'network_session/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-network_session" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Zero Trust Network Session logs", "description": "Collect Zero Trust Network Session logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: network_session_logs\n- name: network_session_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: network_session_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-network_session" ] } ], "template_path": "gcs.yml.hbs", "title": "Zero Trust Network Session logs", "description": "Collect Zero Trust Network Session logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-network_session" ] } ], "template_path": "abs.yml.hbs", "title": "Zero Trust Network Session logs", "description": "Collect Zero Trust Network Session logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "network_session" }, { "type": "logs", "dataset": "cloudflare_logpush.page_shield_events", "title": "Collect Page Shield Event logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/page_shield_events.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/page_shield_events" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-page_shield_events" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Page Shield Event logs", "description": "Collect Page Shield Event logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_page_shield_events", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from. \nThis is only required if you want to collect logs via AWS SQS.\nThis is a Page Shield Event data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "page_shield_events" }, { "name": "cloudflare_r2_page_shield_events", "type": "text", "title": "[Page Shield Event][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3] Number of Workers", "description": "Number of workers that will process the S3 objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'page_shield_events/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-page_shield_events" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Page Shield Event logs", "description": "Collect Page Shield Event logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: page_shield_events_logs\n- name: page_shield_events_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: page_shield_events_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-page_shield_events" ] } ], "template_path": "gcs.yml.hbs", "title": "Page Shield Event logs", "description": "Collect Page Shield Event logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-page_shield_events" ] } ], "template_path": "abs.yml.hbs", "title": "Page Shield Event logs", "description": "Collect Page Shield Event logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "page_shield_events" }, { "type": "logs", "dataset": "cloudflare_logpush.sinkhole_http", "title": "Collect Sinkhole HTTP logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/sinkhole_http.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/sinkhole_http" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-sinkhole_http" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Sinkhole HTTP logs", "description": "Collect Sinkhole HTTP logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_sinkhole_http", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from.\nThis is only required if you want to collect logs via AWS SQS.\nThis is a Sinkhole HTTP data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "sinkhole_http" }, { "name": "cloudflare_r2_sinkhole_http", "type": "text", "title": "[Sinkhole HTTP][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "start_timestamp", "type": "text", "title": "[S3] Start Timestamp", "description": "If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, \"2020-10-10T10:30:00Z\" (UTC) or \"2020-10-10T10:30:00Z+02:30\" (with zone offset).", "multi": false, "required": false, "show_user": false }, { "name": "ignore_older", "type": "text", "title": "[S3] Ignore Older Timespan", "description": "If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.", "multi": false, "required": false, "show_user": false }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'sinkhole_http/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-sinkhole_http" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Sinkhole HTTP logs", "description": "Collect Sinkhole HTTP logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: sinkhole_http_logs\n- name: sinkhole_http_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: sinkhole_http_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-sinkhole_http" ] } ], "template_path": "gcs.yml.hbs", "title": "Sinkhole HTTP logs", "description": "Collect Sinkhole HTTP logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-sinkhole_http" ] } ], "template_path": "abs.yml.hbs", "title": "Sinkhole HTTP logs", "description": "Collect Sinkhole HTTP logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "sinkhole_http" }, { "type": "logs", "dataset": "cloudflare_logpush.spectrum_event", "title": "Collect Spectrum Event logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/spectrum_event.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/spectrum_event" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-spectrum_event" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Spectrum Event Logs", "description": "Collect Spectrum Event logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_spectrum_event", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from. \nThis is only required if you want to collect logs via AWS SQS.\nThis is a spectrum event data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "spectrum_event" }, { "name": "cloudflare_r2_spectrum_event", "type": "text", "title": "[Spectrum Event][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "start_timestamp", "type": "text", "title": "[S3] Start Timestamp", "description": "If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, \"2020-10-10T10:30:00Z\" (UTC) or \"2020-10-10T10:30:00Z+02:30\" (with zone offset).", "multi": false, "required": false, "show_user": false }, { "name": "ignore_older", "type": "text", "title": "[S3] Ignore Older Timespan", "description": "If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.", "multi": false, "required": false, "show_user": false }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. Valid time units are h, m, s.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'spectrum_event/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-spectrum_event" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Spectrum Event Logs", "description": "Collect Spectrum Event logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out. Valid time units are ns, us, ms, s, m, h.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: spectrum_event_logs\n- name: spectrum_event_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: spectrum_event_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-spectrum_event" ] } ], "template_path": "gcs.yml.hbs", "title": "Spectrum Event logs", "description": "Collect Spectrum Event logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-spectrum_event" ] } ], "template_path": "abs.yml.hbs", "title": "Spectrum Event logs", "description": "Collect Spectrum Event logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "spectrum_event" }, { "type": "logs", "dataset": "cloudflare_logpush.workers_trace", "title": "Collect Workers Trace Event logs from Cloudflare", "release": "ga", "ingest_pipeline": "default", "streams": [ { "input": "http_endpoint", "vars": [ { "name": "listen_port", "type": "integer", "title": "Listen Port", "description": "The port number the listener binds to.", "multi": false, "required": true, "show_user": true, "default": 9560 }, { "name": "url", "type": "text", "title": "URL", "description": "This option specifies which URL path to accept requests on. Defaults to /cloudflare_logpush/workers_trace.", "multi": false, "required": false, "show_user": false, "default": "/cloudflare_logpush/workers_trace" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-workers_trace" ] }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false }, { "name": "max_in_flight_bytes", "type": "integer", "title": "Maximum Message Volume", "description": "The limit on total volume of messages from the server that will be allowed at any instant (if zero, no limit). This depends on using the `wait_for_completion_timeout` request query in requests to the server to enable request ACKing. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_max_in_flight_bytes). See the [HTTP Endpoint documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html) for an explanation of how to use `wait_for_completion_timeout`.", "multi": false, "required": false, "show_user": false }, { "name": "retry_after", "type": "integer", "title": "Retry After", "description": "The time in seconds to request the client waits after receiving an over-volume response from the server due to exceeding the Maximum Message Volume. For more details, see [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_retry_after).", "multi": false, "required": false, "show_user": false, "default": 10 }, { "name": "enable_request_tracer", "type": "bool", "title": "Enable request tracing", "description": "The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-http_endpoint.html#_tracer_enabled_3) for details.\n", "multi": false, "required": false, "show_user": false, "default": false } ], "template_path": "http_endpoint.yml.hbs", "title": "Workers Trace Event logs", "description": "Collect Workers Trace Event logs from Cloudflare via HTTP endpoint.", "enabled": true, "ingestion_method": "Webhook" }, { "input": "aws-s3", "vars": [ { "name": "queue_url_workers_trace", "type": "text", "title": "[SQS] Queue URL", "description": "URL of the AWS SQS queue that messages will be received from.\nThis is only required if you want to collect logs via AWS SQS.\nThis is a Workers Trace Event data stream specific queue URL. In order to avoid data loss, do not configure the same SQS queue for more than one data stream.", "multi": false, "required": false, "show_user": true }, { "name": "bucket_list_prefix", "type": "text", "title": "[S3] Bucket Prefix", "description": "Prefix to apply for the list request to the S3 bucket.", "multi": false, "required": false, "show_user": true, "default": "workers_trace" }, { "name": "cloudflare_r2_workers_trace", "type": "text", "title": "[Workers Trace Event][S3] S3-Compatible Bucket Name", "description": "Name of an S3-compatible bucket such as Cloudflare R2, a globally distributed object storage. This parameter can replace Bucket ARN and Access Point ARN with a Bucket Name for collecting logs from Cloudflare R2 or another 3rd party S3-compatible service. It will override the global S3-Compatible Bucket Name if provided.", "multi": false, "required": false, "show_user": true }, { "name": "interval", "type": "text", "title": "[S3] Interval", "description": "Time interval for polling listing of the S3 bucket. NOTE:- Supported units for this parameter are h/m/s.", "multi": false, "required": false, "show_user": true, "default": "1m" }, { "name": "number_of_workers", "type": "integer", "title": "[S3/SQS] Number of Workers", "description": "Number of workers that will process the S3/SQS objects listed.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "start_timestamp", "type": "text", "title": "[S3] Start Timestamp", "description": "If set, only read S3 objects with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, \"2020-10-10T10:30:00Z\" (UTC) or \"2020-10-10T10:30:00Z+02:30\" (with zone offset).", "multi": false, "required": false, "show_user": false }, { "name": "ignore_older", "type": "text", "title": "[S3] Ignore Older Timespan", "description": "If set, ignore S3 objects whose Last-Modified time is before the ignore older timespan. Timespan is checked from the current time to S3 object's Last-Modified time. Accepts a duration like `48h`, `2h30m`.", "multi": false, "required": false, "show_user": false }, { "name": "visibility_timeout", "type": "text", "title": "[SQS] Visibility Timeout", "description": "The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours.", "multi": false, "required": false, "show_user": true, "default": "300s" }, { "name": "api_timeout", "type": "text", "title": "[SQS] API Timeout", "description": "The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "max_number_of_messages", "type": "integer", "title": "[SQS] Maximum Concurrent SQS Messages", "description": "Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.", "multi": false, "required": false, "show_user": true, "default": 5 }, { "name": "file_selectors", "type": "yaml", "title": "[SQS] File Selectors", "description": "If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed.", "multi": false, "required": false, "show_user": false, "default": "- regex: 'workers_trace/'\n" }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-workers_trace" ] }, { "name": "enable_deduplication", "type": "bool", "title": "Enable Data Deduplication", "description": "If data deduplication is enabled, it ensures that no duplicate events are indexed. This is achieved through a unique `_id` generated by the AWS S3 input's [_id Based Deduplication](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-aws-s3#_document_id_generation) feature.", "multi": false, "required": true, "show_user": false, "default": true }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve custom fields for all ECS mappings.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.", "multi": false, "required": false, "show_user": false } ], "template_path": "aws-s3.yml.hbs", "title": "Workers Trace Event logs", "description": "Collect Workers Trace Event logs from Cloudflare via S3 or SQS.", "enabled": true, "ingestion_method": "AWS S3" }, { "input": "gcs", "vars": [ { "name": "alternative_host", "type": "text", "title": "Alternative Host", "description": "Used to override the default host for the storage client (default is storage.googleapis.com).", "multi": false, "required": false, "show_user": false }, { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per bucket.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the bucket will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "bucket_timeout", "type": "text", "title": "Bucket Timeout", "description": "Defines the maximum time that the sdk will wait for a bucket api response before timing out.", "multi": false, "required": false, "show_user": true, "default": "120s" }, { "name": "buckets", "type": "yaml", "title": "Buckets", "description": "This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n", "multi": false, "required": true, "show_user": true, "default": "# You can define as many buckets as you want here.\n\n- name: workers_trace_logs\n- name: workers_trace_logs_2\n\n# The config below is an example of how to override the global config.\n\n#- name: workers_trace_logs_3\n# number_of_workers: 3\n# poll: true\n# poll_interval: 10s\n# bucket_timeout: 30s\n" }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-workers_trace" ] } ], "template_path": "gcs.yml.hbs", "title": "Workers Trace Event logs", "description": "Collect Workers Trace Event logs from Cloudflare via GCS.", "enabled": true, "ingestion_method": "Google Cloud Storage" }, { "input": "azure-blob-storage", "vars": [ { "name": "number_of_workers", "type": "integer", "title": "Maximum number of workers", "description": "Determines how many workers are spawned per container.", "multi": false, "required": false, "show_user": true, "default": 3 }, { "name": "poll", "type": "bool", "title": "Polling", "description": "Determines if the container will be continuously polled for new documents.", "multi": false, "required": false, "show_user": true, "default": true }, { "name": "poll_interval", "type": "text", "title": "Polling interval", "description": "Determines the time interval between polling operations.", "multi": false, "required": false, "show_user": true, "default": "15s" }, { "name": "containers", "type": "yaml", "title": "Containers", "description": "This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. \n", "multi": false, "required": true, "show_user": true, "default": "#- name: azure-container1\n# max_workers: 3\n# poll: true\n# poll_interval: 15s\n#- name: azure-container2\n# max_workers: 3\n# poll: true\n# poll_interval: 10s\n" }, { "name": "file_selectors", "type": "yaml", "title": "File Selectors", "description": "If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false, "default": "# - regex: \"event/\"\n" }, { "name": "timestamp_epoch", "type": "integer", "title": "Timestamp Epoch", "description": "This attribute can be used to filter out files/blobs which have a timestamp older than the specified value. The value of this attribute should be in unix epoch (seconds) format. This process happens locally on the host hence it is an expensive operation. It is recommended to use this attribute only if there is a specific need to filter out files locally.\n", "multi": false, "required": false, "show_user": false }, { "name": "expand_event_list_from_field", "type": "text", "title": "Expand Event List From Field", "description": "If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field).\n", "multi": false, "required": false, "show_user": false }, { "name": "preserve_original_event", "type": "bool", "title": "Preserve original event", "description": "Preserves a raw copy of the original event, added to the field `event.original`.", "multi": false, "required": true, "show_user": true, "default": false }, { "name": "preserve_duplicate_custom_fields", "type": "bool", "title": "Preserve duplicate custom fields", "description": "Preserve github.audit fields that were copied to Elastic Common Schema (ECS) fields.", "multi": false, "required": true, "show_user": false, "default": false }, { "name": "processors", "type": "yaml", "title": "Processors", "description": "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n", "multi": false, "required": false, "show_user": false }, { "name": "tags", "type": "text", "title": "Tags", "description": "Tags to include in the published event.", "multi": true, "required": true, "show_user": false, "default": [ "forwarded", "cloudflare_logpush-workers_trace" ] } ], "template_path": "abs.yml.hbs", "title": "Workers Trace Event logs", "description": "Collect Workers Trace Event logs from Cloudflare via Azure Blob Storage.", "enabled": false, "ingestion_method": "Azure Blob Storage" } ], "package": "cloudflare_logpush", "path": "workers_trace" } ] }