git » sdk » commit 83d5246

diff --git a/borogove/Chat.hx b/borogove/Chat.hx
index e257625..3fa6cd6 100644
--- a/borogove/Chat.hx
+++ b/borogove/Chat.hx
@@ -1971,7 +1971,7 @@ class Channel extends Chat {
 						vcardGet.onFinished(() -> {
 							final vcard = vcardGet.getResult();
 							if (vcard.photo == null) return;
-							persistence.storeMedia(vcard.photo.mime, vcard.photo.data.getData()).then(_ -> {
+							persistence.storeMedia(vcard.photo.mime, vcard.photo.data).then(_ -> {
 								client.trigger("chats/update", [this]);
 							});
 						});
diff --git a/borogove/ChatMessage.hx b/borogove/ChatMessage.hx
index 659d717..e58d406 100644
--- a/borogove/ChatMessage.hx
+++ b/borogove/ChatMessage.hx
@@ -55,6 +55,11 @@ class LinkMetadata {
 @:build(HaxeSwiftBridge.expose())
 #end
 class ChatAttachment {
+	/**
+		Location of the data in cache, if present
+	**/
+	@:allow(borogove)
+	public var cachedAt(default, null): Null<String>;
 	/**
 		Filename
 	**/
diff --git a/borogove/Client.hx b/borogove/Client.hx
index aed6f39..aa30687 100644
--- a/borogove/Client.hx
+++ b/borogove/Client.hx
@@ -415,7 +415,7 @@ class Client extends EventEmitter {
 									brokenAvatars[avatarSha1.toHex()] = from;
 									return;
 								}
-								persistence.storeMedia(vcard.photo.mime ?? "image/png", vcard.photo.data.getData()).then(_ -> {
+								persistence.storeMedia(vcard.photo.mime ?? "image/png", vcard.photo.data).then(_ -> {
 									this.trigger("chats/update", [chat]);
 								});
 							});
@@ -697,7 +697,7 @@ class Client extends EventEmitter {
 							if (item == null) return;
 							final dataNode = item.getChild("data", "urn:xmpp:avatar:data");
 							if (dataNode == null) return;
-							persistence.storeMedia(mime, Base64.decode(StringTools.replace(dataNode.getText(), "\n", "")).getData()).then(_ -> {
+							persistence.storeMedia(mime, Base64.decode(StringTools.replace(dataNode.getText(), "\n", ""))).then(_ -> {
 								this.trigger("chats/update", [chat]);
 							});
 						});
@@ -1108,6 +1108,64 @@ class Client extends EventEmitter {
 		return EventHandled;
 	}
 
+	/**
+		Fetch data for a ChatAttachment into local media cache
+
+		If cachedAt is already filled in for this ChatAttachment, it is simply returned.
+
+		@param attachment ChatAttachment to fetch
+		@returns Promise resolving to a ChatAttachment with cachedAt filled in
+	**/
+	public function fetchAttachment(attachment: ChatAttachment): Promise<ChatAttachment> {
+		// We already have it
+		if (attachment.cachedAt != null) return Promise.resolve(attachment);
+
+		return fetchUris(
+			attachment.uris.copy(),
+			attachment.hashes.find(h -> h.algorithm == "sha-256") ?? attachment.hashes[0]
+		).then(id -> {
+			attachment.cachedAt = id;
+			return attachment;
+		});
+	}
+
+	private function fetchUris(uris: Array<String>, hash: Null<Hash>): Promise<Null<String>> {
+		if (uris.length < 1) return Promise.resolve(null);
+
+		final uri = uris.shift();
+		final aesgcm = XEP0454.parse(uri);
+		if (aesgcm != null) {
+			return XEP0454.fetch(aesgcm, hash).then(
+				data -> persistence.storeMedia(aesgcm.mime, data),
+				e -> fetchUris(uris, hash)
+			);
+		}
+
+		if (uri.startsWith("http://") || uri.startsWith("https://")) {
+			return new Promise((resolve, reject) -> {
+				tink.http.Client.fetch(uri).handle((rOrErr) -> switch (rOrErr) {
+				case Success(r):
+					if (r.header.statusCode != 200) {
+						reject(r.header.statusCode);
+					} else {
+						final mime = switch r.header.contentType() {
+							case Success(ct): ct.toString();
+							default: "application/octet-stream";
+						};
+						resolve({ mime: mime, body: r.body });
+					}
+				case Failure(e):
+					reject(e);
+				});
+			}).then(
+				r -> persistence.storeMedia(r.mime, r.body),
+				e -> fetchUris(uris, hash)
+			);
+		}
+
+		return fetchUris(uris, hash);
+	}
+
 	/**
 		Turn a file into a ChatAttachment for attaching to a ChatMessage
 
@@ -1691,7 +1749,7 @@ class Client extends EventEmitter {
 					if (r == null) {
 						reject("bad or no result from BoB query");
 					} else {
-						persistence.storeMedia(r.type, r.bytes.getData()).then(_ -> resolve(null));
+						persistence.storeMedia(r.type, (r.bytes : Bytes)).then(_ -> resolve(null));
 					}
 				});
 				sendQueryLazy(q);
diff --git a/borogove/Hash.hx b/borogove/Hash.hx
index b96f6e1..5a50ac3 100644
--- a/borogove/Hash.hx
+++ b/borogove/Hash.hx
@@ -1,6 +1,5 @@
 package borogove;
 
-import haxe.crypto.Sha1;
 import haxe.crypto.Sha256;
 import haxe.crypto.Base64;
 import haxe.io.Bytes;
@@ -70,16 +69,33 @@ class Hash {
 		return null;
 	}
 
+	@:allow(borogove)
+	private static function mk(algorithm: String, bytes: Bytes): Null<Hash> {
+		if (algorithm == "sha-1" || algorithm == "sha1") return sha1(bytes);
+		if (algorithm == "sha-256") return sha256(bytes);
+		return null;
+	}
+
 	@:allow(borogove)
 	private static function sha1(bytes: Bytes) {
 		return new Hash("sha-1", Sha1.make(bytes).getData());
 	}
 
+	@:allow(borogove)
+	private static function sha1incr() {
+		return new IncrementalHash("sha-1", new Sha1());
+	}
+
 	@:allow(borogove)
 	private static function sha256(bytes: Bytes) {
 		return new Hash("sha-256", Sha256.make(bytes).getData());
 	}
 
+	@:allow(borogove)
+	private static function sha256incr() {
+		return new IncrementalHash("sha-256", new Sha256());
+	}
+
 	/**
 		Represent this Hash as a URI
 
@@ -129,4 +145,31 @@ class Hash {
 	public function toBase64Url() {
 		return Base64.urlEncode(Bytes.ofData(hash));
 	}
+
+	public function equals(other: Hash) {
+		return serializeUri() == other.serializeUri();
+	}
+}
+
+typedef IncrHash = {
+	function update(b: Bytes): Void;
+	function digest(): Bytes;
+};
+
+class IncrementalHash {
+	public final algorithm: String;
+	public final hash: IncrHash;
+
+	public function new(algorithm: String, hash: IncrHash) {
+		this.algorithm = algorithm;
+		this.hash = hash;
+	}
+
+	public function update(bytes: BytesData) {
+		hash.update(Bytes.ofData(bytes));
+	}
+
+	public function digest() {
+		return new Hash(algorithm, hash.digest().getData());
+	}
 }
diff --git a/borogove/Persistence.hx b/borogove/Persistence.hx
index ea455a7..01767c8 100644
--- a/borogove/Persistence.hx
+++ b/borogove/Persistence.hx
@@ -224,10 +224,10 @@ interface Persistence {
 		Store media bytes and any metadata needed to retrieve them later
 
 		@param mime MIME type of the media
-		@param bytes raw media bytes
-		@returns Promise resolving to true when storage succeeded
+		@param source raw media bytes
+		@returns Promise resolving to an ID for the stored media
 	**/
-	public function storeMedia(mime:String, bytes:BytesData): Promise<Bool>;
+	public function storeMedia(mime:String, source:Source): Promise<String>;
 
 	/**
 		Delete previously stored media
diff --git a/borogove/persistence/Dummy.hx b/borogove/persistence/Dummy.hx
index 9b73588..b40883f 100644
--- a/borogove/persistence/Dummy.hx
+++ b/borogove/persistence/Dummy.hx
@@ -127,8 +127,8 @@ class Dummy implements Persistence {
 	}
 
 	@HaxeCBridge.noemit
-	public function storeMedia(mime:String, bd:BytesData): Promise<Bool> {
-		return Promise.resolve(false);
+	public function storeMedia(mime:String, source:Source): Promise<String> {
+		return Promise.reject("Dummy cannot storeMedia");
 	}
 
 	@HaxeCBridge.noemit
diff --git a/borogove/persistence/MediaStore.hx b/borogove/persistence/MediaStore.hx
index 451acbc..539d048 100644
--- a/borogove/persistence/MediaStore.hx
+++ b/borogove/persistence/MediaStore.hx
@@ -14,7 +14,7 @@ import HaxeCBridge;
 interface MediaStore {
 	public function hasMedia(hashAlgorithm:String, hash:BytesData): Promise<Bool>;
 	public function removeMedia(hashAlgorithm:String, hash:BytesData): Promise<Bool>;
-	public function storeMedia(mime:String, bytes:BytesData): Promise<Bool>;
+	public function storeMedia(mime:String, source:Source): Promise<String>;
 	@:allow(borogove)
 	private function setKV(kv: KeyValueStore):Void;
 }
diff --git a/borogove/persistence/MediaStoreCache.js b/borogove/persistence/MediaStoreCache.js
index 16a60cd..268b9e7 100644
--- a/borogove/persistence/MediaStoreCache.js
+++ b/borogove/persistence/MediaStoreCache.js
@@ -1,6 +1,8 @@
 // This example MediaStore is written in JavaScript
 // so that SDK users can easily see how to write their own
 
+import { borogove_Hash } from "./borogove.js";
+
 export default (cacheName, { routeHashPath } = { routeHashPath: null }) => {
 	let cache = null; // Allow the definitions to be sync
 
@@ -14,13 +16,29 @@ export default (cacheName, { routeHashPath } = { routeHashPath: null }) => {
 			this.kv = kv;
 		},
 
-		async storeMedia(mime, buffer) {
-			const sha256 = await crypto.subtle.digest("SHA-256", buffer);
-			const sha1 = await crypto.subtle.digest("SHA-1", buffer);
-			const sha256NiUrl = mkNiUrl("sha-256", sha256);
-			await cache.put(sha256NiUrl, new Response(buffer, { headers: { "Content-Type": mime } }));
-			if (this.kv) await this.kv.set(mkNiUrl("sha-1", sha1), sha256NiUrl);
-			return true;
+		async storeMedia(mime, source) {
+			const sha256 = borogove_Hash.sha256incr();
+			const sha1 = borogove_Hash.sha1incr();
+			const tmpPath = "/tmp/" + crypto.randomUUID();
+			await cache.put(
+				tmpPath,
+				new Response(source.pipeThrough(new TransformStream({
+					start(controller) {},
+					flush(controller) {},
+					transform(chunk, controller) {
+						sha256.update(chunk);
+						sha1.update(chunk);
+						controller.enqueue(chunk);
+					}
+				})), { headers: { "Content-Type": mime } })
+			);
+			const sha256NiUrl = mkNiUrl("sha-256", sha256.digest().hash);
+			if (this.kv) await this.kv.set(mkNiUrl("sha-1", sha1.digest().hash), sha256NiUrl);
+			// Copy then delete because move is not supported
+			const written = await cache.match(tmpPath);
+			await cache.put(sha256NiUrl, written);
+			await cache.delete(tmpPath);
+			return sha256NiUrl;
 		},
 
 		async removeMedia(hashAlgorithm, hash) {
diff --git a/borogove/persistence/MediaStoreFS.hx b/borogove/persistence/MediaStoreFS.hx
index 0660c35..93fbfc2 100644
--- a/borogove/persistence/MediaStoreFS.hx
+++ b/borogove/persistence/MediaStoreFS.hx
@@ -7,6 +7,8 @@ import haxe.io.Bytes;
 import haxe.io.BytesData;
 import sys.FileSystem;
 import sys.io.File;
+import tink.io.Source;
+import tink.io.Sink;
 import thenshim.Promise;
 
 #if cpp
@@ -76,15 +78,33 @@ class MediaStoreFS implements MediaStore {
 	}
 
 	@HaxeCBridge.noemit
-	public function storeMedia(mime: String, bd: BytesData): Promise<Bool> {
-		final bytes = Bytes.ofData(bd);
-		final sha1 = Hash.sha1(bytes);
-		final sha256 = Hash.sha256(bytes);
-		File.saveBytes(blobpath + "/f" + sha256.toHex(), bytes);
-		return thenshim.PromiseTools.all([
-			set(sha1.serializeUri(), sha256.serializeUri()),
-			set(sha256.serializeUri() + "#contentType", mime)
-		]).then(_ -> true);
+	public function storeMedia(mime: String, source: borogove.Source): Promise<String> {
+		final sha1 = Hash.sha1incr();
+		final sha256 = Hash.sha256incr();
+		final tmpPath = blobpath + "/tmp" + ID.unique();
+		final tmpFile = File.write(tmpPath);
+
+		return new Promise((resolve, reject) -> {
+			((source : RealSource).chunked().map((chunk) -> {
+				sha1.update((chunk : Bytes).getData());
+				sha256.update((chunk : Bytes).getData());
+				return chunk;
+			}) : RealSource).pipeTo(Sink.ofOutput("tmpPath", tmpFile)).handle(o -> switch o {
+				case AllWritten:
+					tmpFile.close();
+					resolve(null);
+				default: reject(o);
+			});
+		}).then(_ -> {
+			final sha1h = sha1.digest();
+			final sha256h = sha256.digest();
+			final path = blobpath + "/f" + sha256h.toHex();
+			sys.FileSystem.rename(tmpPath, path);
+			return thenshim.PromiseTools.all([
+				set(sha1h.serializeUri(), sha256h.serializeUri()),
+				set(sha256h.serializeUri() + "#contentType", mime)
+			]).then(_ -> path);
+		});
 	}
 
 	private function set(k: String, v: Null<String>) {
diff --git a/borogove/persistence/Sqlite.hx b/borogove/persistence/Sqlite.hx
index f87c6e7..1fe033b 100644
--- a/borogove/persistence/Sqlite.hx
+++ b/borogove/persistence/Sqlite.hx
@@ -1004,8 +1004,8 @@ class Sqlite implements Persistence implements KeyValueStore {
 	}
 
 	@HaxeCBridge.noemit
-	public function storeMedia(mime: String, bd: BytesData): Promise<Bool> {
-		return media.storeMedia(mime, bd);
+	public function storeMedia(mime: String, source: Source): Promise<String> {
+		return media.storeMedia(mime, source);
 	}
 
 	@HaxeCBridge.noemit