# Notes ## What do we need to build? - Really, all we need is a unidirecitonal iterator - TODO: check which bits of the flume api patchql uses ## How would we iterate over a go offset? - Open all three files. - Check the journal - (how do we respect locking?) - the journal gives us the number or entries in the log - Start iterating the offset file - map the offset file number to size. - map the size to a slice of bytes by reading that size from the file. Actually, that's not even needed right? - all we do is open the data file, read a u64 as the size, then read that many bytes. Repeat. ## Testing cryptix tried to add tests for ssb messages but failed to understand the .map(type..) stuff on the iterator and got a panic: ``` $ cargo test ... failures: ---- go_offset_log::test::ssb_messages stdout ---- thread 'go_offset_log::test::ssb_messages' panicked at 'called `Result::unwrap()` on an `Err` value: FromUtf8Error { bytes: [134, 166, 65, 117, 116, 104, 111, 114, 130, 164, 65, 108, 103, 111, 167, 101, 100, 50, 53, 53, 49, 57, 162, 73, 68, 218, 0, 32, 252, 86, 210, 174, 217, 193, 106, 79, 195, 74, 31, 175, 157, 175, 85, 245, 187, 143, 225, 99, 213, 34, 94, 251, 142, 150, 30, 254, 168, 193, 223, 82, 163, 75, 101, 121, 130, 164, 65, 108, 103, 111, 166, 115, 104, 97, 50, 53, 54, 164, 72, 97, 115, 104, 218, 0, 32, 187, 110, 22, 14, 1, 75, 74, 100, 92, 125, 185, 28, 161, 22, 127, 164, 235, 84, 144, 227, 238, 69, 81, 5, 59, 181, 32, 35, 20, 75, 150, 173, 168, 80, 114, 101, 118, 105, 111, 117, 115, 192, 163, 82, 97, 119, 218, 1, 156, 123, 10, 32, 32, 34, 112, 114, 101, 118, 105, 111, 117, 115, 34, 58, 32, 110, 117, 108, 108, 44, 10, 32, 32, 34, 97, 117, 116, 104, 111, 114, 34, 58, 32, 34, 64, 47, 70, 98, 83, 114, 116, 110, 66, 97, 107, 47, 68, 83, 104, 43, 118, 110, 97, 57, 86, 57, 98, 117, 80, 52, 87, 80, 86, 73, 108, 55, 55, 106, 112, 89, 101, 47, 113, 106, 66, 51, 49, 73, 61, 46, 101, 100, 50, 53, 53, 49, 57, 34, 44, 10, 32, 32, 34, 115, 101, 113, 117, 101, 110, 99, 101, 34, 58, 32, 49, 44, 10, 32, 32, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 32, 49, 53, 54, 53, 55, 56, 48, 51, 56, 56, 56, 49, 50, 44, 10, 32, 32, 34, 104, 97, 115, 104, 34, 58, 32, 34, 115, 104, 97, 50, 53, 54, 34, 44, 10, 32, 32, 34, 99, 111, 110, 116, 101, 110, 116, 34, 58, 32, 123, 10, 32, 32, 32, 32, 34, 97, 98, 111, 117, 116, 34, 58 , 32, 34, 64, 47, 70, 98, 83, 114, 116, 110, 66, 97, 107, 47, 68, 83, 104, 43, 118, 110, 97, 57, 86, 57, 98, 117, 80, 52, 87, 80, 86, 73, 108, 55, 55, 106, 112, 89, 101, 47, 113, 106, 66, 51, 49, 73, 61, 46, 101, 100, 50, 53, 53, 49, 57, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 116, 101, 115, 116, 32, 117, 115, 101, 114, 34, 44, 10, 32, 32, 32, 32, 34, 116, 121, 112, 101, 34, 58, 32, 34, 97, 98, 111, 117, 116, 34, 10, 32, 32, 125, 44, 10, 32, 32, 34, 115, 105, 103, 110, 97, 116, 117, 114, 101, 34, 58, 32, 34, 51, 73, 99, 99, 71, 56, 116, 83, 54, 83, 70, 106, 80, 98, 114, 76, 122, 105, 81, 53, 98, 121, 69, 47, 110, 80, 101, 43, 84, 98, 105, 52, 98, 115, 121, 99, 48, 49, 82, 84, 53, 98, 69, 114, 52, 79, 122, 76, 113, 65, 114, 55, 121, 112, 111, 48, 104, 113, 74, 114, 79, 120, 116, 79, 67, 88, 106, 80, 120, 97, 88, 104, 121, 87, 117, 85, 72, 47, 106, 116, 48, 69, 80, 77, 67, 103, 61, 61, 46, 115, 105, 103, 46, 101, 100, 50, 53, 53, 49, 57, 34, 10, 125, 168, 83, 101, 113, 117, 101, 110, 99, 101, 1, 169, 84, 105, 109, 101, 115, 116, 97, 109, 112, 168, 193, 244, 94, 88, 93, 83, 233, 164], error: Utf8Error { valid_up_to: 0, error_len: Some(1) } }', src/libcore/result.rs:999:5 note: Run with `RUST_BACKTRACE=1` environment variable to display a backtrace. ``` docoding those bytes in go works though: ```go func TestDecode(t *testing.T) { r := require.New(t) var input = []byte{134, 166, 65, 117, 116, 104, 111, 114, 130, 164, 65, 108, 103, 111, 167, 101, 100, 50, 53, 53, 49, 57, 162, 73, 68, 218, 0, 32, 252, 86, 210, 174, 217, 193, 106, 79, 195, 74, 31, 175, 157, 175, 85, 245, 187, 143, 225, 99, 213, 34, 94, 251, 142, 150, 30, 254, 168, 193, 223, 82, 163, 75, 101, 121, 130, 164, 65, 108, 103, 111, 166, 115, 104, 97, 50, 53, 54, 164, 72, 97, 115, 104, 218, 0, 32, 187, 110, 22, 14, 1, 75, 74, 100, 92, 125, 185, 28, 161, 22, 127, 164, 235, 84, 144, 227, 238, 69, 81, 5, 59, 181, 32, 35, 20, 75, 150, 173, 168, 80, 114, 101, 118, 105, 111, 117, 115, 192, 163, 82, 97, 119, 218, 1, 156, 123, 10, 32, 32, 34, 112, 114, 101, 118, 105, 111, 117, 115, 34, 58, 32, 110, 117, 108, 108, 44, 10, 32, 32, 34, 97, 117, 116, 104, 111, 114, 34, 58, 32, 34, 64, 47, 70, 98, 83, 114, 116, 110, 66, 97, 107, 47, 68, 83, 104, 43, 118, 110, 97, 57, 86, 57, 98, 117, 80, 52, 87, 80, 86, 73, 108, 55, 55, 106, 112, 89, 101, 47, 113, 106, 66, 51, 49, 73, 61, 46, 101, 100, 50, 53, 53, 49, 57, 34, 44, 10, 32, 32, 34, 115, 101, 113, 117, 101, 110, 99, 101, 34, 58, 32, 49, 44, 10, 32, 32, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 32, 49, 53, 54, 53, 55, 56, 48, 51, 56, 56, 56, 49, 50, 44, 10, 32, 32, 34, 104, 97, 115, 104, 34, 58, 32, 34, 115, 104, 97, 50, 53, 54, 34, 44, 10, 32, 32, 34, 99, 111, 110, 116, 101, 110, 116, 34, 58, 32, 123, 10, 32, 32, 32, 32, 34, 97, 98, 111, 117, 116, 34, 58, 32, 34, 64, 47, 70, 98, 83, 114, 116, 110, 66, 97, 107, 47, 68, 83, 104, 43, 118, 110, 97, 57, 86, 57, 98, 117, 80, 52, 87, 80, 86, 73, 108, 55, 55, 106, 112, 89, 101, 47, 113, 106, 66, 51, 49, 73, 61, 46, 101, 100, 50, 53, 53, 49, 57, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 116, 101, 115, 116, 32, 117, 115, 101, 114, 34, 44, 10, 32, 32, 32, 32, 34, 116, 121, 112, 101, 34, 58, 32, 34, 97, 98, 111, 117, 116, 34, 10, 32, 32, 125, 44, 10, 32, 32, 34, 115, 105, 103, 110, 97, 116, 117, 114, 101, 34, 58, 32, 34, 51, 73, 99, 99, 71, 56, 116, 83, 54, 83, 70, 106, 80, 98, 114, 76, 122, 105, 81, 53, 98, 121, 69, 47, 110, 80, 101, 43, 84, 98, 105, 52, 98, 115, 121, 99, 48, 49, 82, 84, 53, 98, 69, 114, 52, 79, 122, 76, 113, 65, 114, 55, 121, 112, 111, 48, 104, 113, 74, 114, 79, 120, 116, 79, 67, 88, 106, 80, 120, 97, 88, 104, 121, 87, 117, 85, 72, 47, 106, 116, 48, 69, 80, 77, 67, 103, 61, 61, 46, 115, 105, 103, 46, 101, 100, 50, 53, 53, 49, 57, 34, 10, 125, 168, 83, 101, 113, 117, 101, 110, 99, 101, 1, 169, 84, 105, 109, 101, 115, 116, 97, 109, 112, 168, 193, 244, 94, 88, 93, 83, 233, 164} var msgt message.StoredMessage c := New(&msgt) v, err := c.Unmarshal(input) r.NoError(err) msg, ok := v.(message.StoredMessage) r.True(ok) t.Log(msg) // Author and Key are not stored a a string in msgpack but a struct with the fields Algo and ID and Algo and Hash (for key) r.Equal("@/FbSrtnBak/DSh+vna9V9buP4WPVIl77jpYe/qjB31I=.ed25519", msg.Author.Ref()) r.Equal("%u24WDgFLSmRcfbkcoRZ/pOtUkOPuRVEFO7UgIxRLlq0=.sha256", msg.Key.Ref()) // Raw should be an opaque byte array rawMessage := `{ "previous": null, "author": "@/FbSrtnBak/DSh+vna9V9buP4WPVIl77jpYe/qjB31I=.ed25519", "sequence": 1, "timestamp": 1565780388812, "hash": "sha256", "content": { "about": "@/FbSrtnBak/DSh+vna9V9buP4WPVIl77jpYe/qjB31I=.ed25519", "name": "test user", "type": "about" }, "signature": "3IccG8tS6SFjPbrLziQ5byE/nPe+Tbi4bsyc01RT5bEr4OzLqAr7ypo0hqJrOxtOCXjPxaXhyWuUH/jt0EPMCg==.sig.ed25519" }` r.Equal(rawMessage, string(msg.Raw)) } ```