Init

Author: tats-u

.github/workflows/master.yml

@@ -0,0 +1,22 @@
+name: CI (master)
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build
+      run: cargo build --verbose
+    - name: Run tests
+      run: cargo test --verbose

.github/workflows/release.yml

@@ -0,0 +1,76 @@
+name: CI (Release)
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  test-again:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+    - uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          target/release
+          target/debug
+          target/.rustc_info.json
+        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+    - name: Test 
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --verbose --all-features
+  release:
+    runs-on: ubuntu-latest
+    needs:
+      - test-again
+    steps:
+    - uses: actions/checkout@v2
+    - name: Create a Release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        # The name of the tag. This should come from the webhook payload, `github.GITHUB_REF` when a user pushes a new tag
+        tag_name: ${{ github.ref }}
+        # The name of the release. For example, `Release v1.0.1`
+        release_name: Release ${{ github.ref }}
+    - name: Get the version
+      id: get_version
+      run: |
+        echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/}
+        echo ::set-output name=DEB_NAME::$(basename $(ls ${{ runner.temp }}/deb-package/*.deb | tail -n 1))
+    - uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          target/release
+          target/debug
+          target/.rustc_info.json
+        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+    - name: Upload crate
+      uses: actions-rs/cargo@v1
+      with:
+        command: publish
+        args: --token ${{ secrets.CARGO_TOKEN }}

.gitignore

@@ -0,0 +1,2 @@
+/target
+Cargo.lock

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "python.formatting.provider": "black",
+    "files.eol": "\n"
+}

Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "oem_cp"
+version = "0.1.0"
+authors = ["Tatsunori Uchino <[email protected]>"]
+edition = "2018"
+categories = ["encoding"]
+keywords = ["encoding", "charset"]
+license = "MIT"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+hashbrown = "0.8.1"
+lazy_static = "1.4.0"

LICENSE

@@ -0,0 +1,7 @@
+Copyright © 2020 Tatsunori Uchino <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,105 @@
+# Rust library for OEM Code pages
+
+This library handles many SBCS (single byte character sets) that are used as OEM code pages. OEM code pages are used for encoding file names in ZIP archives and characters in the terminal in Windows today.
+
+# Supported code pages
+
+| Code Page | Note                                             |
+| --------- | ------------------------------------------------ |
+| 437       | OEM United States                                |
+| 737       | OEM Greek (formerly 437G); Greek (DOS)           |
+| 775       | OEM Baltic; Baltic (DOS)                         |
+| 850       | OEM Multilingual Latin 1; Western European (DOS) |
+| 852       | OEM Latin 2; Central European (DOS)              |
+| 855       | OEM Cyrillic (primarily Russian)                 |
+| 857       | OEM Turkish; Turkish (DOS)                       |
+| 860       | OEM Portuguese; Portuguese (DOS)                 |
+| 861       | OEM Icelandic; Icelandic (DOS)                   |
+| 862       | OEM Hebrew; Hebrew (DOS)                         |
+| 863       | OEM French Canadian; French Canadian (DOS)       |
+| 864       | OEM Arabic; Arabic (864)                         |
+| 865       | OEM Nordic; Nordic (DOS)                         |
+| 866       | OEM Russian; Cyrillic (DOS)                      |
+| 869       | OEM Modern Greek; Greek, Modern (DOS)            |
+| 874       | ANSI/OEM Thai (ISO 8859-11); Thai (Windows)      |
+
+Notes are quoted from https://docs.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
+
+# Examples
+
+## Use specific code pages
+
+### Encoding Unicode string to SBCS bytes
+
+```rust
+use oem_cp::{encoding_string_checked, encoding_string_lossy};
+use oem_cp::code_table::{ENCODING_TABLE_CP437, ENCODING_TABLE_CP737};
+
+assert_eq!(encode_string_checked("π≈22/7", &*ENCODING_TABLE_CP437), Some(vec![0xE3, 0xF7, 0x32, 0x32, 0x2F, 0x37]));
+// Archimedes in Greek
+assert_eq!(encode_string_checked("Αρχιμήδης", &*ENCODING_TABLE_CP737), Some(vec![0x80, 0xA8, 0xAE, 0xA0, 0xA3, 0xE3, 0x9B, 0x9E, 0xAA]));
+// ¾ (U+00BE) is not included in CP437
+assert_eq!(encoding_string_checked("½+¼=¾", &*ENCODING_TABLE_CP437), None);
+// Unknown characters can be replaced with ? (0x3F)
+assert_eq!(encoding_string_lossy("½+¼=¾", &*ENCODING_TABLE_CP437), vec![0xAB, 0x2B, 0xAC, 0x3D, 0x3F]);
+```
+
+### Decoding SBCS bytes to Unicode string
+
+```rust
+use oem_cp::{decode_string_complete_table, decode_string_incomplete_table_checked, decode_string_incomplete_table_lossy};
+use oem_cp::code_table::{DECODING_TABLE_CP437, DECODING_TABLE_CP874};
+
+assert_eq!(&decode_string_complete_table(vec![0xFB, 0xAC, 0x3D, 0xAB], &DECODING_TABLE_CP437), "√¼=½");
+
+// For encoding that has some undefined code points, you must use decode_string_incomplete_table_{checked,lossy} instead of decode_string_complete_table
+
+// means shrimp in Thai (U+E49 => 0xE9)
+assert_eq!(decode_string_incomplete_table_checked(vec![0xA1, 0xD8, 0xE9, 0xA7], &DECODING_TABLE_CP874), Some("กุ้ง".to_string()));
+// 0x81-0x84,0x86-0x90,0x98-0x9F is invalid in CP874
+assert_eq!(decode_string_incomplete_table_checked(vec![0x30, 0x81], &DECODING_TABLE_CP874), None);
+// You can use decode_string_incomplete_table_lossy instead
+assert_eq!(&decode_string_incomplete_table_lossy(vec![0xA1, 0xD8, 0xE9, 0xA7], &DECODING_TABLE_CP874), "กุ้ง");
+// Undefined code points are replaced with U+FFFD (replacement character)
+assert_eq!(&decode_string_incomplete_table_lossy(vec![0x30, 0x81], &DECODING_TABLE_CP874), "0\u{FFFD}");
+```
+
+## Select appropriate codepage from integer
+
+```rust
+use oem_cp::code_table::{ENCODING_TABLE_CP_MAP, DECODING_TABLE_CP_MAP};
+use oem_cp::{encoding_string_checked, encoding_string_lossy};
+
+if let Some(cp874_table) = (*DECODING_TABLE_CP_MAP).get(&874) {
+    assert_eq!(cp874_table.decode_string_checked(vec![0xA1, 0xD8, 0xE9, 0xA7]), Some("กุ้ง".to_string()));
+    // undefined mapping 0x81 for CP874
+    assert_eq!(cp874_table.decode_string_checked(vec![0x81]), None);
+    assert_eq!(&cp874_table.decode_string_lossy(vec![0x81]), "\u{FFFD}");
+} else {
+    panic!("Why the hell CP874 isn't registered?");
+}
+
+if let Some(cp437_table) = (*ENCODING_TABLE_CP_MAP).get(&437) {
+    assert_eq!(encode_string_checked("π≈22/7", cp437_table), Some(vec![0xE3, 0xF7, 0x32, 0x32, 0x2F, 0x37]));
+    // ¾ is undefined in CP437
+    assert_eq!(encoding_string_checked("½+¼=¾", cp437_table), None);
+    // It's replaced with ? (0x3F)
+    assert_eq!(encoding_string_lossy("½+¼=¾", cp437_table), vec![0xAB, 0x2B, 0xAC, 0x3D, 0x3F]);
+} else {
+    panic!("Why the hell CP437 isn't registered?");
+}
+```
+
+# Support for ANSI/EBCDIC/MBCS code pages
+
+For ANSI (125x) and MBCS (932-950; for CJK languages) code pages, please use [encoding_rs](https://github.com/hsivonen/encoding_rs) instead.
+
+This library is only for extended ASCII encodings (0x00-0x80 must be compatible with ASCII), so EBCDIC encodings will never be supported.
+
+# Symbols from 0x01 to 0x19
+
+This library doesn't support [symbols mapped from 0x01 to 0x19 in CP437](https://en.wikipedia.org/wiki/Code_page_437). 0x01-0x19 are mapped to U+0001-U+0019. If you prefer symbols, use [codepage_437](https://github.com/nabijaczleweli/codepage-437) instead.
+
+# Licenses
+
+MIT