/* * QPACK Interop -- encode to intermediate format * * https://github.com/quicwg/base-drafts/wiki/QPACK-Offline-Interop */ #include #if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__NetBSD__) #include #define bswap_16 bswap16 #define bswap_32 bswap32 #define bswap_64 bswap64 #elif defined(__APPLE__) #include #define bswap_16 OSSwapInt16 #define bswap_32 OSSwapInt32 #define bswap_64 OSSwapInt64 #elif defined(WIN32) #define bswap_16 _byteswap_ushort #define bswap_32 _byteswap_ulong #define bswap_64 _byteswap_uint64 #else #include #endif #include #include #include #include #ifdef WIN32 #include #else #include #endif #include #include #include #include #include "lsqpack.h" #include "lsxpack_header.h" static int s_verbose; unsigned char * lsqpack_enc_int (unsigned char *dst, unsigned char *const end, uint64_t value, unsigned prefix_bits); static void usage (const char *name) { fprintf(stderr, "Usage: %s [options] [-i input] [-o output]\n" "\n" "Options:\n" " -i FILE Input file. If not specified or set to `-', the input is\n" " read from stdin.\n" " -o FILE Output file. If not spepcified or set to `-', the output\n" " is written to stdout.\n" " -s NUMBER Maximum number of risked streams. Defaults to %u.\n" " -t NUMBER Dynamic table size. Defaults to %u.\n" " -a MODE Header acknowledgement mode. 0 means headers are never\n" " acknowledged, non-zero means header blocks are acknowledged\n" " immediately. Default value is 0.\n" " -n Process annotations.\n" " -S Server mode.\n" " -D Do not emit \"Duplicate\" instructions.\n" " -A Aggressive indexing.\n" " -M Turn off memory guard.\n" " -f Fast: use maximum output buffers.\n" " -v Verbose: print various messages to stderr.\n" "\n" " -h Print this help screen and exit\n" , name, LSQPACK_DEF_MAX_RISKED_STREAMS, LSQPACK_DEF_DYN_TABLE_SIZE); } static void write_enc_stream (FILE *out, const unsigned char *enc_buf, size_t enc_sz) { uint64_t stream_id_enc; uint32_t length_enc; size_t written; if (enc_sz <= 0) return; stream_id_enc = 0; length_enc = enc_sz; #if __BYTE_ORDER == __LITTLE_ENDIAN stream_id_enc = bswap_64(stream_id_enc); length_enc = bswap_32(length_enc); #endif written = fwrite(&stream_id_enc, 1, sizeof(stream_id_enc), out); if (written != sizeof(stream_id_enc)) { perror("fwrite"); exit(EXIT_FAILURE); } written = fwrite(&length_enc, 1, sizeof(length_enc), out); if (written != sizeof(length_enc)) { perror("fwrite"); exit(EXIT_FAILURE); } written = fwrite(enc_buf, 1, enc_sz, out); if (written != enc_sz) { perror("fwrite"); exit(EXIT_FAILURE); } } static void write_enc_and_header_streams (FILE *out, unsigned stream_id, const unsigned char *enc_buf, size_t enc_sz, const unsigned char *pref_buf, size_t pref_sz, const unsigned char *hea_buf, size_t hea_sz) { uint64_t stream_id_enc; uint32_t length_enc; size_t written; if (s_verbose) fprintf(stderr, "%s: stream %"PRIu32"\n", __func__, stream_id); if (enc_sz) { stream_id_enc = 0; length_enc = enc_sz; #if __BYTE_ORDER == __LITTLE_ENDIAN stream_id_enc = bswap_64(stream_id_enc); length_enc = bswap_32(length_enc); #endif written = fwrite(&stream_id_enc, 1, sizeof(stream_id_enc), out); if (written != sizeof(stream_id_enc)) goto write_err; written = fwrite(&length_enc, 1, sizeof(length_enc), out); if (written != sizeof(length_enc)) goto write_err; written = fwrite(enc_buf, 1, enc_sz, out); if (written != enc_sz) goto write_err; } stream_id_enc = stream_id; length_enc = pref_sz + hea_sz; #if __BYTE_ORDER == __LITTLE_ENDIAN stream_id_enc = bswap_64(stream_id_enc); length_enc = bswap_32(length_enc); #endif written = fwrite(&stream_id_enc, 1, sizeof(stream_id_enc), out); if (written != sizeof(stream_id_enc)) goto write_err; written = fwrite(&length_enc, 1, sizeof(length_enc), out); if (written != sizeof(length_enc)) goto write_err; written = fwrite(pref_buf, 1, pref_sz, out); if (written != pref_sz) goto write_err; written = fwrite(hea_buf, 1, hea_sz, out); if (written != hea_sz) goto write_err; return; write_err: perror("fwrite"); exit(EXIT_FAILURE); } static unsigned s_saved_ins_count; static int ack_last_entry_id (struct lsqpack_enc *encoder) { unsigned char *end_cmd; unsigned char cmd[80]; unsigned val; if (s_verbose) fprintf(stderr, "ACK entry ID %u\n", encoder->qpe_ins_count); cmd[0] = 0x00; val = encoder->qpe_ins_count - s_saved_ins_count; s_saved_ins_count = encoder->qpe_ins_count; end_cmd = lsqpack_enc_int(cmd, cmd + sizeof(cmd), val, 6); assert(end_cmd > cmd); return lsqpack_enc_decoder_in(encoder, cmd, end_cmd - cmd); } static int ack_stream (struct lsqpack_enc *encoder, uint64_t stream_id) { unsigned char *end_cmd; unsigned char cmd[80]; if (s_verbose) fprintf(stderr, "ACK stream ID %"PRIu64"\n", stream_id); cmd[0] = 0x80; end_cmd = lsqpack_enc_int(cmd, cmd + sizeof(cmd), stream_id, 7); assert(end_cmd > cmd); return lsqpack_enc_decoder_in(encoder, cmd, end_cmd - cmd); } static int sync_table (struct lsqpack_enc *encoder, uint64_t num_inserts) { unsigned char *end_cmd; unsigned char cmd[80]; if (s_verbose) fprintf(stderr, "Sync table num inserts %"PRIu64"\n", num_inserts); cmd[0] = 0x00; end_cmd = lsqpack_enc_int(cmd, cmd + sizeof(cmd), num_inserts, 6); assert(end_cmd > cmd); return lsqpack_enc_decoder_in(encoder, cmd, end_cmd - cmd); } static int cancel_stream (struct lsqpack_enc *encoder, uint64_t stream_id) { unsigned char *end_cmd; unsigned char cmd[80]; if (s_verbose) fprintf(stderr, "Cancel stream ID %"PRIu64"\n", stream_id); cmd[0] = 0x40; end_cmd = lsqpack_enc_int(cmd, cmd + sizeof(cmd), stream_id, 6); assert(end_cmd > cmd); return lsqpack_enc_decoder_in(encoder, cmd, end_cmd - cmd); } int main (int argc, char **argv) { FILE *in = stdin; FILE *out = stdout; int opt; unsigned dyn_table_size = LSQPACK_DEF_DYN_TABLE_SIZE, max_risked_streams = LSQPACK_DEF_MAX_RISKED_STREAMS; unsigned lineno, stream_id; struct lsqpack_enc encoder; char *line, *end, *tab; ssize_t pref_sz; enum lsqpack_enc_status st; enum lsqpack_enc_opts enc_opts = 0; size_t enc_sz, hea_sz, enc_off, hea_off; int header_opened, r; unsigned arg; enum { ACK_NEVER, ACK_IMMEDIATE, } ack_mode = ACK_NEVER; int process_annotations = 0; char line_buf[0x1000]; unsigned char tsu_buf[LSQPACK_LONGEST_SDTC]; size_t tsu_buf_sz; enum lsqpack_enc_header_flags hflags; int fast = 0; struct lsxpack_header xhdr; unsigned char enc_buf[0x1000], hea_buf[0x1000], pref_buf[0x20]; while (-1 != (opt = getopt(argc, argv, "ADMSa:i:no:s:t:hvf"))) { switch (opt) { case 'S': enc_opts |= LSQPACK_ENC_OPT_SERVER; break; case 'D': enc_opts |= LSQPACK_ENC_OPT_NO_DUP; break; case 'A': enc_opts |= LSQPACK_ENC_OPT_IX_AGGR; break; case 'M': enc_opts |= LSQPACK_ENC_OPT_NO_MEM_GUARD; break; case 'n': ++process_annotations; break; case 'a': ack_mode = atoi(optarg) ? ACK_IMMEDIATE : ACK_NEVER; break; case 'i': if (0 != strcmp(optarg, "-")) { in = fopen(optarg, "r"); if (!in) { fprintf(stderr, "cannot open `%s' for reading: %s\n", optarg, strerror(errno)); exit(EXIT_FAILURE); } } break; case 'o': if (0 != strcmp(optarg, "-")) { out = fopen(optarg, "wb"); if (!out) { fprintf(stderr, "cannot open `%s' for writing: %s\n", optarg, strerror(errno)); exit(EXIT_FAILURE); } } break; case 's': max_risked_streams = atoi(optarg); break; case 't': dyn_table_size = atoi(optarg); break; case 'h': usage(argv[0]); exit(EXIT_SUCCESS); case 'f': fast = 1; break; case 'v': ++s_verbose; break; default: exit(EXIT_FAILURE); } } tsu_buf_sz = sizeof(tsu_buf); if (0 != lsqpack_enc_init(&encoder, s_verbose ? stderr : NULL, dyn_table_size, dyn_table_size, max_risked_streams, enc_opts, tsu_buf, &tsu_buf_sz)) { perror("lsqpack_enc_init"); exit(EXIT_FAILURE); } lineno = 0; stream_id = 0; enc_off = 0; hea_off = 0; header_opened = 0; while (line = fgets(line_buf, sizeof(line_buf), in), line != NULL) { ++lineno; end = strchr(line, '\n'); if (!end) { fprintf(stderr, "no newline on line %u\n", lineno); exit(EXIT_FAILURE); } *end = '\0'; if (end == line) { if (header_opened) { size_t sz, pref_max = sizeof(pref_buf); for (sz = (fast ? pref_max : 0); sz <= pref_max; sz++) { pref_sz = lsqpack_enc_end_header(&encoder, pref_buf, sz, &hflags); if (pref_sz > 0) { if (max_risked_streams == 0) assert(!(hflags & LSQECH_REF_AT_RISK)); break; } } assert(pref_sz <= lsqpack_enc_header_block_prefix_size(&encoder)); if (pref_sz < 0) { fprintf(stderr, "end_header failed: %s", strerror(errno)); exit(EXIT_FAILURE); } if (ack_mode == ACK_IMMEDIATE) { if (!(2 == pref_sz && pref_buf[0] == 0 && pref_buf[1] == 0)) r = ack_stream(&encoder, stream_id); else r = 0; if (r == 0 && encoder.qpe_ins_count > s_saved_ins_count) r = ack_last_entry_id(&encoder); else r = 0; if (r != 0) { fprintf(stderr, "acking stream %u failed: %s", stream_id, strerror(errno)); exit(EXIT_FAILURE); } } if (s_verbose) fprintf(stderr, "compression ratio: %.3f\n", lsqpack_enc_ratio(&encoder)); write_enc_and_header_streams(out, stream_id, enc_buf, enc_off, pref_buf, pref_sz, hea_buf, hea_off); enc_off = 0; hea_off = 0; header_opened = 0; } continue; } if (*line == '#') { if (!process_annotations) continue; /* Lines starting with ## are potential annotations */ if (ack_mode != ACK_IMMEDIATE /* Ignore ACK annotations in immediate ACK mode, as we do * not tolerate duplicate ACKs. */ && 1 == sscanf(line, "## %*[a] %u ", &arg)) { if (0 != ack_stream(&encoder, arg)) { fprintf(stderr, "ACKing stream ID %u failed\n", arg); exit(EXIT_FAILURE); } } else if (1 == sscanf(line, "## %*[s] %u", &arg)) sync_table(&encoder, arg); else if (1 == sscanf(line, "## %*[c] %u", &arg)) cancel_stream(&encoder, arg); else if (1 == sscanf(line, "## %*[t] %u", &arg)) { tsu_buf_sz = sizeof(tsu_buf); if (0 != lsqpack_enc_set_max_capacity(&encoder, arg, tsu_buf, &tsu_buf_sz)) { fprintf(stderr, "cannot set capacity to %u: %s\n", arg, strerror(errno)); exit(EXIT_FAILURE); } write_enc_stream(out, tsu_buf, tsu_buf_sz); } continue; } tab = strchr(line, '\t'); if (!tab) { fprintf(stderr, "no TAB on line %u\n", lineno); exit(EXIT_FAILURE); } if (!header_opened) { ++stream_id; if (0 != lsqpack_enc_start_header(&encoder, stream_id, 0)) { fprintf(stderr, "start_header failed: %s\n", strerror(errno)); exit(EXIT_FAILURE); } header_opened = 1; } if (fast) { enc_sz = sizeof(enc_buf) - enc_off; hea_sz = sizeof(hea_buf) - hea_off; } else { /* Increase buffers one by one to exercise error conditions */ enc_sz = 0; hea_sz = 0; } while (1) { lsxpack_header_set_offset2(&xhdr, line, 0, tab - line, tab + 1 - line, end - tab - 1); st = lsqpack_enc_encode(&encoder, enc_buf + enc_off, &enc_sz, hea_buf + hea_off, &hea_sz, &xhdr, 0); switch (st) { case LQES_NOBUF_ENC: if (enc_sz < sizeof(enc_buf) - enc_off) ++enc_sz; else assert(0); break; case LQES_NOBUF_HEAD: if (hea_sz < sizeof(hea_buf) - hea_off) ++hea_sz; else assert(0); break; default: assert(st == LQES_OK); goto end_encode_one_header; } } if (st != LQES_OK) { /* It could only run of of output space, so it's not really an * error, but we make no provision in the interop encoder to * grow the buffers. */ fprintf(stderr, "Could not encode header on line %u: %u\n", lineno, st); exit(EXIT_FAILURE); } end_encode_one_header: enc_off += enc_sz; hea_off += hea_sz; } if (s_verbose) fprintf(stderr, "exited while loop\n"); (void) fclose(in); if (header_opened) { if (s_verbose) fprintf(stderr, "close opened header\n"); pref_sz = lsqpack_enc_end_header(&encoder, pref_buf, sizeof(pref_buf), &hflags); if (pref_sz < 0) { fprintf(stderr, "end_header failed: %s", strerror(errno)); exit(EXIT_FAILURE); } if (max_risked_streams == 0) assert(!(hflags & LSQECH_REF_AT_RISK)); if (ack_mode == ACK_IMMEDIATE && !(2 == pref_sz && pref_buf[0] == 0 && pref_buf[1] == 0) && 0 != ack_stream(&encoder, stream_id)) { fprintf(stderr, "acking stream %u failed: %s", stream_id, strerror(errno)); exit(EXIT_FAILURE); } if (s_verbose) fprintf(stderr, "compression ratio: %.3f\n", lsqpack_enc_ratio(&encoder)); write_enc_and_header_streams(out, stream_id, enc_buf, enc_off, pref_buf, pref_sz, hea_buf, hea_off); } lsqpack_enc_cleanup(&encoder); if (0 != fclose(out)) { perror("fclose(out)"); exit(EXIT_FAILURE); } exit(EXIT_SUCCESS); }