Public
Open-source lightweight git hosting.
text_encoding_test.go
·
100 lines
·
2.9 KB
| 1 | package git |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/binary" |
| 6 | "os" |
| 7 | "os/exec" |
| 8 | "path/filepath" |
| 9 | "testing" |
| 10 | "unicode/utf16" |
| 11 | ) |
| 12 | |
| 13 | func utf16Bytes(s string, order binary.ByteOrder) []byte { |
| 14 | units := utf16.Encode([]rune(s)) |
| 15 | out := make([]byte, 0, len(units)*2) |
| 16 | for _, u := range units { |
| 17 | var pair [2]byte |
| 18 | order.PutUint16(pair[:], u) |
| 19 | out = append(out, pair[:]...) |
| 20 | } |
| 21 | return out |
| 22 | } |
| 23 | |
| 24 | func TestNormalizeTextEncoding(t *testing.T) { |
| 25 | utf8BOM := append([]byte{0xEF, 0xBB, 0xBF}, []byte("# readme")...) |
| 26 | if got := string(normalizeTextEncoding(utf8BOM)); got != "# readme" { |
| 27 | t.Fatalf("utf-8 BOM: got %q", got) |
| 28 | } |
| 29 | |
| 30 | le := append([]byte{0xFF, 0xFE}, utf16Bytes("# readme \u2713", binary.LittleEndian)...) |
| 31 | if got := string(normalizeTextEncoding(le)); got != "# readme \u2713" { |
| 32 | t.Fatalf("utf-16le: got %q", got) |
| 33 | } |
| 34 | |
| 35 | be := append([]byte{0xFE, 0xFF}, utf16Bytes("# readme \u2713", binary.BigEndian)...) |
| 36 | if got := string(normalizeTextEncoding(be)); got != "# readme \u2713" { |
| 37 | t.Fatalf("utf-16be: got %q", got) |
| 38 | } |
| 39 | |
| 40 | plain := []byte("# plain readme") |
| 41 | if got := string(normalizeTextEncoding(plain)); got != "# plain readme" { |
| 42 | t.Fatalf("plain text must stay untouched, got %q", got) |
| 43 | } |
| 44 | |
| 45 | binaryData := []byte{0x00, 0x01, 0x02, 0x03} |
| 46 | if got := normalizeTextEncoding(binaryData); len(got) != len(binaryData) { |
| 47 | t.Fatalf("binary data must stay untouched") |
| 48 | } |
| 49 | } |
| 50 | |
| 51 | // TestReadBlobDecodesUTF16 mirrors README files written by Windows tooling |
| 52 | // (PowerShell writes UTF-16LE by default) and expects them to render as text. |
| 53 | func TestReadBlobDecodesUTF16(t *testing.T) { |
| 54 | ctx := context.Background() |
| 55 | tmp := t.TempDir() |
| 56 | barePath := filepath.Join(tmp, "repo.git") |
| 57 | if err := InitBare(ctx, barePath, "main"); err != nil { |
| 58 | t.Fatalf("InitBare: %v", err) |
| 59 | } |
| 60 | |
| 61 | work := filepath.Join(tmp, "work") |
| 62 | if err := os.MkdirAll(work, 0o755); err != nil { |
| 63 | t.Fatalf("mkdir: %v", err) |
| 64 | } |
| 65 | run := func(args ...string) { |
| 66 | t.Helper() |
| 67 | cmd := exec.CommandContext(ctx, "git", args...) |
| 68 | cmd.Dir = work |
| 69 | if out, err := cmd.CombinedOutput(); err != nil { |
| 70 | t.Fatalf("git %v: %v (%s)", args, err, string(out)) |
| 71 | } |
| 72 | } |
| 73 | run("init", "-b", "main") |
| 74 | run("config", "user.name", "Tester") |
| 75 | run("config", "user.email", "tester@example.com") |
| 76 | |
| 77 | content := append([]byte{0xFF, 0xFE}, utf16Bytes("# Windows README\n", binary.LittleEndian)...) |
| 78 | if err := os.WriteFile(filepath.Join(work, "README.md"), content, 0o644); err != nil { |
| 79 | t.Fatalf("write README: %v", err) |
| 80 | } |
| 81 | run("add", ".") |
| 82 | run("commit", "-m", "utf-16 readme") |
| 83 | run("push", barePath, "main") |
| 84 | |
| 85 | blob, err := ReadBlob(ctx, barePath, "main", "README.md") |
| 86 | if err != nil { |
| 87 | t.Fatalf("ReadBlob: %v", err) |
| 88 | } |
| 89 | if blob.IsBinary { |
| 90 | t.Fatalf("BOM-marked UTF-16 file must not be treated as binary") |
| 91 | } |
| 92 | if string(blob.Data) != "# Windows README\n" { |
| 93 | t.Fatalf("unexpected decoded content %q", string(blob.Data)) |
| 94 | } |
| 95 | |
| 96 | readme, err := FindReadme(ctx, barePath, "main", "") |
| 97 | if err != nil || readme == nil { |
| 98 | t.Fatalf("FindReadme should find the UTF-16 README, err=%v", err) |
| 99 | } |
| 100 | } |
| 101 |