It looks like maybe the only way to create Data from ârawâ bytes is Data.fromBase64String.
btoa: Base64 encoding
In browsers, the btoa (MDN) function (Binary to (Base64) Ascii) is the built-in way to do Base64 encoding. Its input format is a bit wonky (a string of code points between U+0 and U+FF, representing byte values), but your functions already produce what it needs (the for loop through the .join('')). btoa takes this input string and produces the Base64 encoding of the series of represented bytes.
Scriptable has a btoa function, but I ran into a problem when I tried to use it to âround-tripâ full-range byte data.
testB64('Scriptable btoa U+41', () => btoa('A'), 'QQ==', [65]); // ok
testB64('Scriptable btoa U+80', () => btoa('\u{80}'), 'gA==', [128]); // woA= [194,128]
testB64('Scriptable btoa U+100', () => btoa('\u{100}'), /* no expected values, should throw */); // xIA= [196,128]
testB64
/**
* @param {string} m
* @param {()=>string} fn
* @param {string|undefined} eB64
* @param {number[]|undefined} eBytes
*/
function testB64(m, fn, eB64 = undefined, eBytes = undefined) {
let b64;
try {
b64 = fn();
} catch (e) {
if (eB64)
console.error(`ERROR: ${m} expected ${eB64}, but threw error ${String(e)}`);
else
console.log(`OK: ${m} threw error ${String(e)}`);
return;
}
const b64Ok = b64 == eB64;
if (!b64Ok)
console.error(`ERROR: ${m}: expected ${eB64 ?? 'an error'}, got ${b64}`);
const bytes = Data.fromBase64String(b64).getBytes();
const bytesOk = eBytes && eq(eBytes, bytes)
if (!bytesOk)
console.error(`ERROR: ${m}: expected ${eBytes ? `[${eBytes}]` : 'an error'}, got [${bytes}]`);
if (b64Ok && bytesOk)
console.log(`OK: ${m}`)
/**
* @param {number[]} a
* @param {number[]} b
*/
function eq(a, b) {
if (a.length != b.length) return false;
return a.every((v, i) => v == b[i]);
}
}
It looks like the problem is that the input string is encoded as UTF-8 before being Base64 encoded. In my testing, any code point higher than U+7F is represented by multiple bytes (that match its UTF-8 encoding), and code points over U+FF are not rejected like they âshouldâ be (browsers and Node btoa throw an error).
WebView to the rescue?
If we ship the data over to a WebView its btoa works as expected, but we have to do it asynchronously.
const wv = new WebView;
/**
* @param {string} s
* @returns {Promise<string>}
*/
function webBtoa(s) {
return wv.evaluateJavaScript(`btoa(${JSON.stringify(s)})`);
}
await testAsyncB64('WebView btoa U+41', () => webBtoa('A'), 'QQ==', [65]);
await testAsyncB64('WebView btoa U+80', () => webBtoa('\u{80}'), 'gA==', [128]);
await testAsyncB64('WebView btoa U+100', () => webBtoa('\u{100}'), /* no expected values, should throw */);
testB64 and testAsyncB64
/**
* @param {string} m
* @param {()=>string} fn
* @param {string|undefined} eB64
* @param {number[]|undefined} eBytes
*/
function testB64(m, fn, eB64 = undefined, eBytes = undefined) {
let b64;
try {
b64 = fn();
} catch (e) {
if (eB64)
console.error(`ERROR: ${m} expected ${eB64}, but threw error ${String(e)}`);
else
console.log(`OK: ${m} threw error ${String(e)}`);
return;
}
const b64Ok = b64 == eB64;
if (!b64Ok)
console.error(`ERROR: ${m}: expected ${eB64 ?? 'an error'}, got ${b64}`);
const bytes = Data.fromBase64String(b64).getBytes();
const bytesOk = eBytes && eq(eBytes, bytes)
if (!bytesOk)
console.error(`ERROR: ${m}: expected ${eBytes ? `[${eBytes}]` : 'an error'}, got [${bytes}]`);
if (b64Ok && bytesOk)
console.log(`OK: ${m}`)
/**
* @param {number[]} a
* @param {number[]} b
*/
function eq(a, b) {
if (a.length != b.length) return false;
return a.every((v, i) => v == b[i]);
}
}
/**
* @param {string} m
* @param {()=>Promise<string>} fn
* @param {string|undefined} eB64
* @param {number[]|undefined} eBytes
* @returns {Promise<void>}
*/
async function testAsyncB64(m, fn, eB64 = undefined, eBytes = undefined) {
try {
const b64 = await fn();
return void testB64(m, () => b64, eB64, eBytes);
} catch (e) {
if (eB64)
console.error(`ERROR: ${m} expected ${eB64}, but threw error ${String(e)}`);
else
console.log(`OK: ${m} threw error ${String(e)}`);
return;
}
}
Here is an ar2data function and tests that show it encodes (with WebView btoa) and decodes (with Data .getBytes) arbitrary byte data properly:
/**
* @param {number[]} ar
* @returns {Promise<Data>}
*/
async function ar2data(ar) {
return Data.fromBase64String(await webBtoa(ar.map(b => String.fromCodePoint(b)).join('')));
}
// const ar = Data.fromFile(pathname).getBytes()
const ar = new Array(256).fill(0).map((b, i) => i); // fake data: 0-255 values
const rt = (await ar2data(ar)).getBytes();
showEq('round-tripped 0-255', ar, rt);
const arPlus = ar.map(b => (b * 3) % 256); // all 0-255 byte values, but different order
const rtPlus = (await ar2data(arPlus)).getBytes();
showEq('round-tripped (0-255)*3 mod 256', arPlus, rtPlus);
webBtoa and showEq
const wv = new WebView;
/**
* @param {string} s
* @returns {Promise<string>}
*/
function webBtoa(s) {
return wv.evaluateJavaScript(`btoa(${JSON.stringify(s)})`);
}
/**
* @param {string} m
* @param {number[]} a
* @param {number[]} b
*/
function showEq(m, a, b) {
if (a.length != b.length)
return void console.error(`${m} length mismatch: ${a.length} vs ${b.length}`)
for (let i = 0; i < a.length; i++)
if (a[i] != b[i])
return void console.error(`${m} mismatch at ${i}: ${a[i]} vs ${b[i]}`);
console.log(`${m} ok`);
}
Once you have a Data object from (e.g.) ar2data, you should be able to write it to a file with a FileManager like your first .write call. I was able to generate Data from my âall bytesâ blobs, write them to a temporary file, and verified that I got the same byte sequences back after reading into a new Data object from FileManager.
Copy/Re-implement btoa?
If you donât want to deal with asynchronous code, or maybe using a WebView is too much overhead, you could probably integrate a âpure JSâ btoa implementation. npm.js shows several under btoa or base64. MDN links to btoa in core-js that looks like it wouldnât be too hard to rework into a standalone function. Just be sure to follow any licensing that applies.
Typed Array
Oh, and you mentioned Typed Arrays. It looks like .getBytes() returns a plain array, not the Uint8Array. This probably doesnât matter in most cases. Your for loops would work fine with Type Arrays, but my use of .map to convert byte values to strings would fail (Typed Array .map produces a new Typed Array, which canât hold strings; Array.from() could help by creating a new plain array from the typed one).
const a = [];
console.log(`( [] ) gives ${a.constructor.name}`); // Array
const a2 = new Array;
console.log(`( new Array ) gives ${a2.constructor.name}`); // Array
const b = Data.fromString('').getBytes();
console.log(`( [Data obj].getBytes() ) gives ${b.constructor.name}`); // Array
const u8 = new Uint8Array;
console.log(`( new Uint8Array ) gives ${u8.constructor.name}`); // Uint8Array