// Cargo.toml
[dependencies]
postcard = "1.0"
serde = { version = "1.0", default-features = false, features = ["derive"] }

// Example usage
use postcard::{to_vec, from_bytes};
use serde::{Serialize, Deserialize};

#[derive(Serialize, Deserialize)]
struct SensorReading {
    timestamp: u64,
    temperature: f32,
    humidity: f32,
    pressure: f32,
}

fn serialize_readings(readings: &[SensorReading]) -> Vec<u8> {
    postcard::to_allocvec(readings).unwrap()
}

// Ideal for sensor data with regular intervals
// Timestamps: 10:00:00, 10:00:01, 10:00:02... -> nearly free
// Temperatures: 23.5, 23.5, 23.6, 23.5... -> highly compressible

// Cargo.toml
[dependencies]
serde = { version = "1.0", default-features = false, features = ["derive"] }
serde_cbor = { version = "0.11", default-features = false }

// Usage identical to JSON but binary output
let data = SensorReading { ... };
let bytes = serde_cbor::to_vec(&data)?;

use parquet::basic::{Compression, Encoding};
use parquet::data_type::{FloatType, Int64Type};
use parquet::file::properties::WriterProperties;
use parquet::file::writer::SerializedFileWriter;
use parquet::schema::parser::parse_message_type;
use std::fs::File;
use std::sync::Arc;

fn write_parquet_file(readings: &[SensorReading], filename: &str) -> Result<()> {
    let message_type = "
        message sensor_data {
            required int64 timestamp (TIMESTAMP_MILLIS);
            required float temperature;
            required float humidity;
            required float pressure;
            // ... more columns
        }
    ";

    let schema = Arc::new(parse_message_type(message_type)?);

    // Snappy compression - pure Rust, works on ESP32!
    let props = WriterProperties::builder()
        .set_compression(Compression::SNAPPY)
        .set_encoding(Encoding::PLAIN)
        .set_statistics_enabled(parquet::file::properties::EnabledStatistics::None)
        .build();

    let file = File::create(filename)?;
    let mut writer = SerializedFileWriter::new(file, schema, Arc::new(props))?;
    let mut row_group_writer = writer.next_row_group()?;

    // Write each column using typed() API
    {
        let mut col_writer = row_group_writer.next_column()?.unwrap();
        col_writer
            .typed::<Int64Type>()
            .write_batch(&timestamps, None, None)?;
        col_writer.close()?;
    }

    // Float columns
    {
        let mut col_writer = row_group_writer.next_column()?.unwrap();
        col_writer
            .typed::<FloatType>()
            .write_batch(&temperatures, None, None)?;
        col_writer.close()?;
    }

    row_group_writer.close()?;
    writer.close()?;
    Ok(())
}

use rusty_s3::{Bucket, Credentials, S3Action, UrlStyle};
use std::time::Duration;

fn generate_s3_upload_url(object_key: &str, credentials: &Credentials) -> String {
    let bucket = Bucket::new(
        "https://s3.us-west-2.amazonaws.com".parse().unwrap(),
        UrlStyle::VirtualHost,
        "your-bucket",
        "us-west-2",
    ).unwrap();

    let mut put_action = bucket.put_object(Some(credentials), object_key);
    put_action.headers_mut().insert("content-type", "application/octet-stream");

    // Sign URL - valid for 1 hour
    put_action.sign(Duration::from_secs(3600)).to_string()
}

use esp_idf_svc::http::client::{EspHttpConnection, Configuration};
use embedded_svc::http::client::Client;

fn upload_to_s3(signed_url: &str, parquet_data: &[u8]) -> Result<(), Error> {
    let config = Configuration::default();
    let mut client = Client::wrap(EspHttpConnection::new(&config)?);

    let headers = [("Content-Type", "application/octet-stream")];
    let mut request = client.put(signed_url, &headers)?;
    request.write_all(parquet_data)?;

    let response = request.submit()?;
    if response.status() == 200 {
        log::info!("Upload successful!");
    }
    Ok(())
}

import boto3
import pyarrow as pa
import pyarrow.parquet as pq
import postcard  # or cbor2

def lambda_handler(event, context):
    # Download from S3
    s3 = boto3.client('s3')
    obj = s3.get_object(Bucket='raw-bucket', Key=event['key'])
    compressed_data = obj['Body'].read()

    # Deserialize (Postcard/CBOR/custom)
    readings = postcard.loads(compressed_data)

    # Convert to Arrow Table
    table = pa.Table.from_pydict({
        'timestamp': [r['timestamp'] for r in readings],
        'temperature': [r['temperature'] for r in readings],
        'humidity': [r['humidity'] for r in readings],
    })

    # Write as Parquet with compression
    pq.write_table(table, '/tmp/output.parquet', compression='snappy')

    # Upload to optimized bucket
    s3.upload_file('/tmp/output.parquet', 'parquet-bucket',
                   f'{event["key"]}.parquet')

use arrow::array::*;
use arrow::datatypes::*;
use parquet::arrow::ArrowWriter;

// Deserialize from Postcard/CBOR
let readings: Vec<SensorReading> = postcard::from_bytes(&data)?;

// Build Arrow arrays
let timestamps = UInt64Array::from(
    readings.iter().map(|r| r.timestamp).collect::<Vec<_>>()
);
let temps = Float32Array::from(
    readings.iter().map(|r| r.temperature).collect::<Vec<_>>()
);

// Create schema
let schema = Schema::new(vec![
    Field::new("timestamp", DataType::UInt64, false),
    Field::new("temperature", DataType::Float32, false),
]);

// Write Parquet
let file = File::create("output.parquet")?;
let mut writer = ArrowWriter::try_new(file, Arc::new(schema), None)?;
writer.write(&batch)?;
writer.close()?;

# Install espup (manages ESP Rust toolchains)
cargo install espup

# Install toolchains for your target chip
espup install

# For ESP32-S3 (Xtensa) - installs fork automatically
# For ESP32-C6 (RISC-V) - uses standard toolchain

# Source the environment (add to .bashrc/.zshrc)
source ~/export-esp.sh

# Install project generator
cargo install esp-generate

# Generate new project
esp-generate --chip=esp32s3 my-sensor-project

# Or for ESP32-C6
esp-generate --chip=esp32c6 my-sensor-project

# Install flash tool
cargo install espflash

# Build and flash
cd my-sensor-project
cargo build --release
espflash flash --monitor target/xtensa-esp32s3-espidf/release/my-sensor-project

use esp_idf_svc::{
    wifi::{EspWifi, ClientConfiguration, Configuration},
    eventloop::EspSystemEventLoop,
    nvs::EspDefaultNvsPartition,
};

fn connect_wifi() -> Result<EspWifi<'static>, Error> {
    let sys_loop = EspSystemEventLoop::take()?;
    let nvs = EspDefaultNvsPartition::take()?;

    let mut wifi = EspWifi::new(peripherals.modem, sys_loop, Some(nvs))?;

    wifi.set_configuration(&Configuration::Client(ClientConfiguration {
        ssid: "YourSSID".try_into().unwrap(),
        password: "YourPassword".try_into().unwrap(),
        ..Default::default()
    }))?;

    wifi.start()?;
    wifi.connect()?;

    Ok(wifi)
}

use esp_idf_svc::http::client::{EspHttpConnection, Configuration};
use embedded_svc::http::client::Client;

fn upload_to_s3(presigned_url: &str, data: &[u8]) -> Result<(), Error> {
    let config = Configuration::default();
    let mut client = Client::wrap(EspHttpConnection::new(&config)?);

    let headers = [("Content-Type", "application/octet-stream")];

    let mut request = client.put(presigned_url, &headers)?;
    request.write_all(data)?;

    let response = request.submit()?;

    if response.status() == 200 {
        log::info!("Upload successful!");
    }

    Ok(())
}

use postcard::to_allocvec;
use serde::{Serialize, Deserialize};

#[derive(Serialize, Deserialize)]
struct SensorReading {
    timestamp: u64,
    temperature: f32,
    humidity: f32,
}

fn collect_and_upload(readings: Vec<SensorReading>, presigned_url: &str) -> Result<(), Error> {
    // Serialize with Postcard (tiny binary format)
    let data = to_allocvec(&readings)?;

    log::info!("Serialized {} readings to {} bytes", readings.len(), data.len());

    // Upload to S3
    upload_to_s3(presigned_url, &data)?;

    Ok(())
}