Crates.io | datafusion-server |
lib.rs | datafusion-server |
version | 0.17.1 |
source | src |
created_at | 2023-09-20 12:57:26.566286 |
updated_at | 2024-07-13 02:03:09.995855 |
description | Web server library for session-based queries using Arrow and other large datasets as data sources. |
homepage | https://github.com/sal-openlab/datafusion-server |
repository | https://github.com/sal-openlab/datafusion-server.git |
max_upload_size | |
id | 978105 |
size | 273,696 |
Multiple session, variety of data sources query server implemented by Rust.
Please see the Documentation for an introductory tutorial and a full usage guide. Additionally, the REST API documentation is available according to the OpenAPI specification. Also, refer to the CHANGELOG for the latest information.
License under the MIT
Copyright (c) 2022 - 2024 SAL Ltd. - https://sal.co.jp
and other LLVM supported environment.
$ docker pull ghcr.io/sal-openlab/datafusion-server/datafusion-server:latest
or built without Python plugin version.
$ docker pull ghcr.io/sal-openlab/datafusion-server/datafusion-server-without-plugin:latest
$ docker run -d --rm \
-p 4000:4000 \
-v ./data:/var/datafusion-server/data \
--name datafusion-server \
ghcr.io/sal-openlab/datafusion-server/datafusion-server:latest
If you are only using sample data in a container, omit the -v ./data:/var/xapi-server/data
.
$ cd <repository-root-dir>
$ ./make-containers.sh
$ docker run -d --rm \
-p 4000:4000 \
-v ./bin/data:/var/datafusion-server/data \
--name datafusion-server \
datafusion-server:0.18.1
If you are only using sample data in a container, omit the -v ./bin/data:/var/xapi-server/data
.
$ cargo init server-executor
$ cd server-executor
[package]
name = "server-executor"
version = "0.1.0"
edition = "2021"
[dependencies]
datafusion-server = "0.18.1"
use std::path::PathBuf;
use clap::Parser;
use datafusion_server::settings::Settings;
#[derive(Parser)]
#[clap(author, version, about = "Arrow and other large datasets web server", long_about = None)]
struct Args {
#[clap(
long,
value_parser,
short = 'f',
value_name = "FILE",
help = "Configuration file",
default_value = "./config.toml"
)]
config: PathBuf,
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let args = Args::parse();
let settings = Settings::new_with_file(&args.config)?;
datafusion_server::execute(settings)?;
Ok(())
}
For details, further reading main.rs and Config.toml.
# Configuration file of datafusion-server
[server]
port = 4000
flight_grpc_port = 50051
base_url = "/"
data_dir = "./data"
plugin_dir = "./plugins"
[session]
default_keep_alive = 3600 # in seconds
upload_limit_size = 20 # MB
[log]
# trace, debug, info, warn, error
level = "debug"
$ cargo run
Require Python interpreter v3.7+
[package]
name = "server-executor"
version = "0.1.0"
edition = "2021"
[dependencies]
datafusion-server = { version = "0.18.1", features = ["plugin"] }
$ cargo run
[package]
name = "server-executor"
version = "0.1.0"
edition = "2021"
[profile.release]
opt-level = 'z'
strip = true
lto = "fat"
codegen-units = 1
[dependencies]
datafusion-server = { version = "0.18.1", features = ["plugin"] }
$ cargo build --release
$ cargo clean
$ curl -X "POST" "http://localhost:4000/dataframe/query" \
-H 'Content-Type: application/json' \
-d $'
{
"dataSources": [
{
"format": "csv",
"name": "sales",
"location": "file:///superstore.csv",
"options": {
"inferSchemaRows": 100,
"hasHeader": true
}
}
],
"query": {
"sql": "SELECT * FROM sales"
},
"response": {
"format": "json"
}
}'
$ curl -X "POST" "http://localhost:4000/dataframe/query" \
-H 'Content-Type: application/json' \
-H 'Accept: text/csv' \
-d $'
{
"dataSources": [
{
"format": "json",
"name": "population",
"location": "https://datausa.io/api/data?drilldowns=State&measures=Population",
"options": {
"jsonPath": "$.data[*]"
}
}
],
"query": {
"sql": "SELECT * FROM population WHERE \"ID Year\">=2020"
}
}'
$ curl -X "POST" "http://localhost:4000/dataframe/query" \
-H 'Content-Type: application/json' \
-H 'Accept: application/json' \
-d $'
{
"dataSources": [
{
"format": "arrow",
"name": "example",
"location": "excel://example-workbook.xlsx/Sheet1",
"pluginOptions": {
"skipRows": 2
}
}
],
"query": {
"sql": "SELECT * FROM example"
}
}'