diff --git a/app/application.ml b/app/application.ml index c6994f3..7c8ae74 100644 --- a/app/application.ml +++ b/app/application.ml @@ -146,6 +146,7 @@ type timeline_notice = | Delete_failed of string | Status_failed of string | Copy_failed of string + | Import_failed of string let operation_failure = function | Some (Delete_failed message) -> @@ -160,6 +161,12 @@ let operation_failure = function , "The block keeps its current status. Open its Status menu to try again." ) | Some (Copy_failed message) -> Some ("Copy failed", message, "Select Copy to try again.") + | Some (Import_failed message) -> + Some + ( "Attachment not added" + , message + , "Your journal entry and other attachments are kept. Select the file again to \ + retry." ) | None | Some Delete_undo -> None ;; @@ -257,6 +264,7 @@ type state = ; asset_import_request : int ; asset_import_owner : (string * int64) option ; capture_pick_request : int + ; capture_picker_armed : bool ; capture_pick_source : Journal_asset_import.source ; capture_imports : capture_import_batch option ; capture_error : capture_failure option @@ -343,6 +351,7 @@ let initial_state = ; asset_import_request = 0 ; asset_import_owner = None ; capture_pick_request = 0 + ; capture_picker_armed = false ; capture_pick_source = Journal_asset_import.Files ; capture_imports = None ; capture_error = None @@ -416,15 +425,22 @@ let track_capture_session state = else { state with next_local_sequence = next } ;; -let clear_graph_surface state = +let clear_graph_surface ?(discard_capture = true) state = (match state.direct_capture with - | Some capture -> + | Some capture when discard_capture -> List.iter Journal_asset_import.discard_staged_file (Journal_capture.pending_attachments capture) - | None -> ()); + | None | Some _ -> ()); { state with - favorites = + capture_picker_armed = false + ; capture_pick_request = + (state.capture_pick_request + + + if Option.is_some state.direct_capture || Option.is_some state.draft_graph + then 1 + else 0) + ; favorites = Journal_routes.Favorites.create ~graph_generation:state.graph_state.generation ; favorites_media_roots = Rrbvec.empty ; favorites_requests = [] @@ -461,6 +477,15 @@ let discard_local_graph_state state = ;; let clear_account_drafts state = + Graph_drafts.iter + (fun _ draft -> + Option.iter + (fun capture -> + List.iter + Journal_asset_import.discard_staged_file + (Journal_capture.pending_attachments capture)) + draft.capture_draft) + state.graph_drafts; { (clear_graph_surface state) with draft_graph = None ; graph_drafts = Graph_drafts.empty @@ -490,7 +515,9 @@ let switch_draft_graph state graph_id = } state.graph_drafts in - let state = clear_graph_surface state in + let state = + clear_graph_surface ~discard_capture:(Option.is_none state.draft_graph) state + in let retained = Option.bind graph_id (fun id -> Graph_drafts.find_opt (Logseq_db_types.Graph_types.Uuid.to_string id) graph_drafts) @@ -823,6 +850,7 @@ let back_state state = let apply_worker_response_unstaged state (response : Journal_graph_runtime.response) = match response.payload with + | Projection_changes _ | Projection_resync -> state | Reference_sources_changed updates -> let reference_sources = List.fold_left @@ -1234,6 +1262,8 @@ module Root_navigation = struct | Capture_discarded | Capture_picker_requested of Journal_asset_import.source | Capture_asset_picked of Journal_asset_import.staged * int option + | Capture_assets_picked of + Journal_asset_import.staged list * int option * string option | Capture_native_edit of Ui.Event.Payload.text_edit | Capture_task_intent of bool | Capture_edited of string @@ -1299,23 +1329,79 @@ module Root_navigation = struct modal = No_modal ; direct_capture = None ; capture_error = None + ; capture_picker_armed = false ; capture_pick_request = state.capture_pick_request + 1 } | None | Some _ -> state) | Capture_picker_requested source -> { state with - capture_pick_source = source + capture_picker_armed = true + ; capture_pick_source = source ; capture_pick_request = state.capture_pick_request + 1 } - | Capture_asset_picked (staged, request_id) -> + | (Capture_asset_picked (_, _) as event) + | (Capture_assets_picked (_, _, _) as event) -> + let items, request_id, error = + match event with + | Capture_asset_picked (staged, request_id) -> [ staged ], request_id, None + | Capture_assets_picked (items, request_id, error) -> items, request_id, error + | _ -> assert false + in + let owned = + Option.fold + ~none:[] + ~some:Journal_capture.pending_attachments + state.direct_capture + in + let discard_from owned item = + if + not + (List.exists + (fun pick -> + String.equal + (Journal_asset_import.staged_path pick) + (Journal_asset_import.staged_path item)) + owned) + then Journal_asset_import.discard_staged_file item + in (match state.direct_capture with - | Some capture when request_id = Some state.capture_pick_request -> + | Some capture + when request_id = Some state.capture_pick_request + && (state.capture_picker_armed + || + match event with + | Capture_asset_picked _ -> true + | _ -> false) -> + let capture, rejected = + List.fold_left + (fun (capture, rejected) staged -> + let next = Journal_capture.add_attachment capture staged in + if next == capture + then ( + discard_from (Journal_capture.pending_attachments capture) staged; + capture, rejected || not (Journal_capture.can_attach capture)) + else next, rejected) + (capture, false) + items + in + let error = + match error with + | Some _ -> error + | None when rejected -> + Some "Some selections could not be added to this draft." + | None -> None + in { state with - direct_capture = Some (Journal_capture.add_attachment capture staged) - ; capture_error = None + direct_capture = Some capture + ; capture_picker_armed = + (match event with + | Capture_assets_picked _ -> false + | _ -> state.capture_picker_armed) + ; capture_error = + Option.map (fun message -> Local_capture_failure message) error } | None | Some _ -> - Journal_asset_import.discard_staged_file staged; + List.iter (discard_from owned) items; state) | Capture_native_edit edit -> (match state.direct_capture with @@ -3264,6 +3350,7 @@ type app_context = ; send_action : action -> unit ; apply_platform : bytes -> unit Effect.t ; running : bool ref + ; release_staging : unit -> unit } let latest_patch = ref "" @@ -3372,12 +3459,16 @@ let start let copy_owner = ref None in let copy_sequence = ref 0L in let copy_worker_requests = Hashtbl.create 2 in + let projection_request_accepted = ref (fun _ -> ()) in + let projection_request_finished = ref (fun _ -> ()) in let cancel_copy () = let state, _ = Journal_graph_runtime.Copy.step !copy_state Cancel in copy_state := state; copy_owner := None; Hashtbl.iter - (fun request_id _ -> Worker.cancel client ~request_id) + (fun request_id (_, _, protocol_id) -> + !projection_request_finished protocol_id; + Worker.cancel client ~request_id) copy_worker_requests in let copy_current owner = @@ -3437,6 +3528,7 @@ let start in match Worker.send client (Graph_service.Graph_request request) with | Accepted request_id -> + !projection_request_accepted request; Hashtbl.replace copy_worker_requests request_id @@ -3529,6 +3621,8 @@ let start let import_worker_requests = Hashtbl.create 2 in (* operation token -> staged pick whose temp copy is removed on completion *) let capture_staged_items = Hashtbl.create 2 in + let capture_import_queue = Queue.create () in + let drain_capture_imports = ref (fun () -> Effect.ignore) in let asset_worker_requests = Hashtbl.create 2 in let asset_runtime = Journal_asset_runtime.create @@ -3548,11 +3642,31 @@ let start | Worker.Accepted worker_id -> (match request with | Graph_service.Graph_request request -> - Hashtbl.replace asset_worker_requests worker_id request.request_id + sync_media !state_ref; + Journal_media_runtime.observe_request media_runtime request; + Hashtbl.replace + asset_worker_requests + worker_id + (media_key !state_ref, request.request_id) | _ -> ()); true | Full | Not_ready | Stopping -> false) in + let observe_projection_request request = + sync_media !state_ref; + Journal_media_runtime.observe_request media_runtime request; + Journal_asset_runtime.observe_request asset_runtime request + in + let forget_projection_request request_id = + Journal_media_runtime.forget_request media_runtime request_id; + Journal_asset_runtime.forget_request asset_runtime ~request_id + in + projection_request_accepted := observe_projection_request; + projection_request_finished := forget_projection_request; + let observe_projection_response response = + Journal_media_runtime.observe_response media_runtime response; + Journal_asset_runtime.observe_response asset_runtime response + in let asset_settings = ref None in let refresh_assets ~graph_generation calendar = Option.iter @@ -3565,11 +3679,6 @@ let start (Option.bind calendar (fun calendar -> Option.map (fun settings -> calendar, settings) !asset_settings)) in - let invalidate_assets () = - Journal_media_runtime.refresh media_runtime; - if !state_ref.graph_state.phase = Graph_open - then Journal_asset_runtime.invalidate asset_runtime - in let started_graph_generation = ref None in let send_manager command = Effect.of_thunk (fun () -> @@ -3581,12 +3690,24 @@ let start terminal_graph_state state (Transport_graph_error message) in let graph_worker_requests = Hashtbl.create 16 in - let deliver_output output = + let deliver_output (output : Journal_graph_runtime.output) = + List.iter + (fun response -> + match response.Journal_graph_runtime.payload with + | Projection_changes windows -> + Journal_media_runtime.changes media_runtime windows; + Journal_asset_runtime.changes asset_runtime windows + | Projection_resync -> + Journal_media_runtime.resync media_runtime; + Journal_asset_runtime.resync asset_runtime + | _ -> ()) + output.responses; Journal_graph_transport.deliver ~runtime:graph_runtime ~send:(fun request -> match Worker.send client (Graph_service.Graph_request request) with | Accepted worker_id -> + observe_projection_request request; Hashtbl.replace graph_worker_requests worker_id request; Journal_graph_transport.Accepted | Full -> Full @@ -3609,6 +3730,7 @@ let start Worker.send client (Graph_service.Graph_request protocol_request) with | Accepted worker_request_id -> + observe_projection_request protocol_request; Hashtbl.replace admission_worker_requests worker_request_id @@ -3733,8 +3855,8 @@ let start Journal_graph_runtime.reset graph_runtime; Hashtbl.clear graph_worker_requests; Hashtbl.clear favorites_worker_requests; + Hashtbl.clear admission_worker_requests; let current = !state_ref in - refresh_assets ~graph_generation:graph_state.generation current.calendar; let graph_info = Journal_graph_runtime.start graph_runtime in let feed_generation = current.next_request_generation in let feed_output = @@ -3789,7 +3911,11 @@ let start in Effect.bind prepare ~f:(fun () -> Effect.bind - (Effect.of_thunk (fun () -> deliver_output output)) + (Effect.of_thunk (fun () -> + refresh_assets + ~graph_generation:graph_state.generation + !state_ref.calendar; + deliver_output output)) ~f:(fun delivery -> set_state (fun state -> apply_delivery_responses state delivery)))) | Graph_closed | Graph_opening | Graph_closing | Graph_failed -> @@ -3872,8 +3998,14 @@ let start | Completed (Graph_service.Graph_response (Logseq_db_worker.Protocol.V2_response response)) when Logseq_db_types.Graph_types.Uuid.equal response.request_id protocol_id -> + if copy_current owner + then + observe_projection_response (Logseq_db_worker.Protocol.V2_response response) + else forget_projection_request protocol_id; Journal_graph_runtime.Copy.Completed (ticket, response.outcome) - | _ -> Read_failed (ticket, "The copy request was cancelled or failed.") + | _ -> + forget_projection_request protocol_id; + Read_failed (ticket, "The copy request was cancelled or failed.") in copy_transition owner event | Worker.Response { request_id; outcome = Completed response; _ } @@ -3889,7 +4021,6 @@ let start Journal_media_runtime.reject media_runtime ticket; flush_media set_state | Worker.Push { payload = Graph_service.Graph_push push; _ } -> - invalidate_assets (); let snapshot = !state_ref in let admission_refresh = trigger_admission @@ -3940,24 +4071,60 @@ let start ; _ } when Hashtbl.mem asset_worker_requests worker_id -> + let context, protocol_id = Hashtbl.find asset_worker_requests worker_id in Hashtbl.remove asset_worker_requests worker_id; - ignore (Journal_asset_runtime.receive asset_runtime response : bool); + let (Logseq_db_worker.Protocol.V2_response { request_id = actual; _ }) = response in + if context = media_key !state_ref && Option.is_some context + then + if + Logseq_db_types.Graph_types.Uuid.equal actual protocol_id + && Journal_asset_runtime.receive asset_runtime response + then Journal_media_runtime.observe_response media_runtime response + else ( + forget_projection_request protocol_id; + Journal_asset_runtime.reject asset_runtime ~request_id:protocol_id); Effect.ignore | Worker.Response { request_id ; outcome = Worker.Completed (Graph_service.Graph_response response) ; _ } -> - (match response, Hashtbl.find_opt graph_worker_requests request_id with - | ( Logseq_db_worker.Protocol.V2_response - { request_id = actual; outcome = V2_resync_required _; _ } - , Some expected ) - when expected.request_id = actual -> invalidate_assets () - | _ -> ()); + let expected = + match Hashtbl.find_opt graph_worker_requests request_id with + | Some request -> Some request + | None -> + (match Hashtbl.find_opt admission_worker_requests request_id with + | Some (_, request) -> Some request + | None -> + Option.map snd (Hashtbl.find_opt favorites_worker_requests request_id)) + in + let (Logseq_db_worker.Protocol.V2_response { request_id = actual; _ }) = response in + let owned = + match expected with + | Some request + when Logseq_db_types.Graph_types.Uuid.equal request.request_id actual -> + observe_projection_response response; + true + | Some request -> + forget_projection_request request.request_id; + false + | None -> false + in Hashtbl.remove graph_worker_requests request_id; Hashtbl.remove admission_worker_requests request_id; Hashtbl.remove favorites_worker_requests request_id; - let output = Journal_graph_runtime.receive graph_runtime response in + let output = + if owned + then Journal_graph_runtime.receive graph_runtime response + else ( + match expected with + | Some request -> + Journal_graph_runtime.fail_request + graph_runtime + request + ~message:"Worker response did not match the accepted request" + | None -> Journal_graph_runtime.{ requests = []; responses = [] }) + in Effect.bind (Effect.of_thunk (fun () -> deliver_output output)) ~f:(fun delivery -> @@ -4002,7 +4169,7 @@ let start let refresh_after_worker_event = let (Logseq_db_worker.Protocol.V2_response { outcome; _ }) = response in match outcome with - | V2_mutation_committed _ -> + | V2_mutation_committed _ when owned -> let snapshot = !state_ref in trigger_admission set_state_and_effect @@ -4041,62 +4208,71 @@ let start Journal_asset_import.discard_staged_file staged | None -> ()) | None -> ()); - let current = - match pending with - | Some (generation, _, owner) -> - let snapshot = !state_ref in - generation = snapshot.graph_state.generation - && - let routes = - Option.bind owner (fun (entry_id, request_generation) -> - Option.bind - (Journal_routes.at_entry snapshot.routes ~entry_id) - (fun routes -> - if Journal_routes.detail_request_generation routes = request_generation - then Some routes - else None)) - in - (match result, Option.bind routes Journal_routes.detail with - | Ok receipt, Some detail -> - Journal_model.id (Journal_detail.root detail) - = Logseq_db_types.Graph_types.Uuid.to_string receipt.target - | Error _, _ -> true - | _ -> false) - | None -> false + let completion = + let current = + match pending with + | Some (generation, _, owner) -> + let snapshot = !state_ref in + generation = snapshot.graph_state.generation + && + let routes = + Option.bind owner (fun (entry_id, request_generation) -> + Option.bind + (Journal_routes.at_entry snapshot.routes ~entry_id) + (fun routes -> + if + Journal_routes.detail_request_generation routes + = request_generation + then Some routes + else None)) + in + (match result, Option.bind routes Journal_routes.detail with + | Ok receipt, Some detail -> + Journal_model.id (Journal_detail.root detail) + = Logseq_db_types.Graph_types.Uuid.to_string receipt.target + | Error _, _ -> true + | _ -> false) + | None -> false + in + Effect.bind + (Effect.of_thunk (fun () -> + sync_media !state_ref; + Result.iter (Journal_media_runtime.imported media_runtime ~current) result)) + ~f:(fun () -> + Effect.many + [ flush_media set_state + ; (match pending with + | None -> Effect.ignore + | Some (generation, operation, owner) -> + set_state (fun state -> + if + state.graph_state.generation <> generation + || Option.fold + ~none:false + ~some:(fun (entry_id, request_generation) -> + match Journal_routes.at_entry state.routes ~entry_id with + | Some routes -> + Journal_routes.detail_request_generation routes + <> request_generation + | None -> true) + owner + then state + else + { state with + timeline_notice = + (match owner, result with + | None, Error message -> Some (Import_failed message) + | _ -> state.timeline_notice) + ; import_completion = + Some + ( operation + , match result with + | Ok _ -> None + | Error message -> Some message ) + })) + ]) in - Effect.bind - (Effect.of_thunk (fun () -> - sync_media !state_ref; - Result.iter (Journal_media_runtime.imported media_runtime ~current) result)) - ~f:(fun () -> - Effect.many - [ flush_media set_state - ; (match pending with - | None -> Effect.ignore - | Some (generation, operation, owner) -> - set_state (fun state -> - if - state.graph_state.generation <> generation - || Option.fold - ~none:false - ~some:(fun (entry_id, request_generation) -> - match Journal_routes.at_entry state.routes ~entry_id with - | Some routes -> - Journal_routes.detail_request_generation routes - <> request_generation - | None -> true) - owner - then state - else - { state with - import_completion = - Some - ( operation - , match result with - | Ok _ -> None - | Error message -> Some message ) - })) - ]) + Effect.bind completion ~f:(fun () -> !drain_capture_imports ()) | Worker.Response { request_id; outcome = Failed _ | Cancelled | Shutdown; _ } when Hashtbl.mem import_worker_requests request_id -> let generation, operation, owner = Hashtbl.find import_worker_requests request_id in @@ -4106,23 +4282,31 @@ let start Hashtbl.remove capture_staged_items operation; Journal_asset_import.discard_staged_file staged | None -> ()); - set_state (fun state -> - if - state.graph_state.generation <> generation - || Option.fold - ~none:false - ~some:(fun (entry_id, request_generation) -> - match Journal_routes.at_entry state.routes ~entry_id with - | Some routes -> - Journal_routes.detail_request_generation routes <> request_generation - | None -> true) - owner - then state - else - { state with - import_completion = - Some (operation, Some "Import was interrupted. Select the file again.") - }) + let completion = + set_state (fun state -> + if + state.graph_state.generation <> generation + || Option.fold + ~none:false + ~some:(fun (entry_id, request_generation) -> + match Journal_routes.at_entry state.routes ~entry_id with + | Some routes -> + Journal_routes.detail_request_generation routes <> request_generation + | None -> true) + owner + then state + else + { state with + timeline_notice = + (match owner with + | None -> + Some (Import_failed "Import was interrupted. Select the file again.") + | Some _ -> state.timeline_notice) + ; import_completion = + Some (operation, Some "Import was interrupted. Select the file again.") + }) + in + Effect.bind completion ~f:(fun () -> !drain_capture_imports ()) | Worker.Response { outcome = Completed (Asset_file _); _ } -> Effect.ignore | Worker.Response { outcome = Completed Client_command_completed; _ } -> Effect.ignore | Worker.Response { outcome = Completed (Graph_state graph_state); _ } @@ -4154,13 +4338,17 @@ let start | None | Some _ -> state) | Worker.Response { request_id; outcome = Failed _ | Cancelled | Shutdown; _ } when Hashtbl.mem asset_worker_requests request_id -> - let protocol_id = Hashtbl.find asset_worker_requests request_id in + let context, protocol_id = Hashtbl.find asset_worker_requests request_id in Hashtbl.remove asset_worker_requests request_id; - Journal_asset_runtime.reject asset_runtime ~request_id:protocol_id; + if context = media_key !state_ref && Option.is_some context + then ( + forget_projection_request protocol_id; + Journal_asset_runtime.reject asset_runtime ~request_id:protocol_id); Effect.ignore | Worker.Response { request_id; outcome = Failed _ | Cancelled | Shutdown; _ } when Hashtbl.mem favorites_worker_requests request_id -> let request, protocol_request = Hashtbl.find favorites_worker_requests request_id in + forget_projection_request protocol_request.request_id; Journal_graph_runtime.abandon graph_runtime protocol_request; Hashtbl.remove favorites_worker_requests request_id; set_state (fun state -> @@ -4170,6 +4358,7 @@ let start | Worker.Response { request_id; outcome = Failed _ | Cancelled | Shutdown; _ } when Hashtbl.mem admission_worker_requests request_id -> let request, protocol_request = Hashtbl.find admission_worker_requests request_id in + forget_projection_request protocol_request.request_id; Journal_graph_runtime.abandon graph_runtime protocol_request; Hashtbl.remove admission_worker_requests request_id; update_admission set_state_and_effect (fun state -> @@ -4182,6 +4371,7 @@ let start when Hashtbl.mem graph_worker_requests request_id -> let request = Hashtbl.find graph_worker_requests request_id in Hashtbl.remove graph_worker_requests request_id; + forget_projection_request request.request_id; let message = match outcome with | Failed message -> message @@ -4452,6 +4642,7 @@ let start Worker.send client (Graph_service.Graph_request protocol_request) with | Accepted worker_request_id -> + observe_projection_request protocol_request; Hashtbl.replace favorites_worker_requests worker_request_id @@ -4528,55 +4719,70 @@ let start | _ -> state))) in let prev_capture_imports_key = ref initial_state.capture_imports in + let rec drain_imports () = + if Hashtbl.length capture_staged_items > 0 || Queue.is_empty capture_import_queue + then Effect.ignore + else ( + let generation, target, staged = Queue.take capture_import_queue in + let operation = Journal_asset_import.staged_token staged in + let failure = + if + (not !running) + || (not !state_ref.write_enabled) + || !state_ref.graph_state.generation <> generation + then Some "The destination is no longer ready for imports." + else ( + match + Worker.send + client + (Graph_service.Import_asset + { graph_generation = generation + ; source = Journal_asset_import.to_import staged ~target + }) + with + | Accepted request_id -> + Hashtbl.replace import_worker_requests request_id (generation, operation, None); + Hashtbl.replace capture_staged_items operation staged; + None + | Full | Not_ready | Stopping -> + Some "Import is temporarily unavailable. Select the file again.") + in + match failure with + | None -> Effect.ignore + | Some message -> + Journal_asset_import.discard_staged_file staged; + Effect.bind + (set_state (fun state -> + if state.graph_state.generation <> generation + then state + else + { state with + import_completion = Some (operation, Some message) + ; timeline_notice = Some (Import_failed message) + })) + ~f:drain_imports) + in + drain_capture_imports := drain_imports; let capture_imports_callback batch = match batch with | None -> Effect.ignore | Some batch -> - Effect.many + Effect.bind (set_state (fun state -> match state.capture_imports with | Some pending when pending == batch -> { state with capture_imports = None } - | _ -> state) - :: - (match Logseq_db_types.Graph_types.Uuid.of_string batch.batch_target with + | _ -> state)) + ~f:(fun () -> + match Logseq_db_types.Graph_types.Uuid.of_string batch.batch_target with | Error _ -> - [ Effect.of_thunk (fun () -> - List.iter Journal_asset_import.discard_staged_file batch.batch_items) - ] + List.iter Journal_asset_import.discard_staged_file batch.batch_items; + Effect.ignore | Ok target -> - List.map + List.iter (fun staged -> - let operation = Journal_asset_import.staged_token staged in - let source = Journal_asset_import.to_import staged ~target in - Effect.bind - (Effect.of_thunk (fun () -> - if not !state_ref.write_enabled - then ( - Journal_asset_import.discard_staged_file staged; - Some "The destination is not ready for imports") - else ( - match - Worker.send - client - (Graph_service.Import_asset - { graph_generation = batch.batch_generation; source }) - with - | Accepted request_id -> - Hashtbl.replace - import_worker_requests - request_id - (batch.batch_generation, operation, None); - Hashtbl.replace capture_staged_items operation staged; - None - | Full | Not_ready | Stopping -> - Journal_asset_import.discard_staged_file staged; - Some "Import is temporarily unavailable. Select the file again."))) - ~f:(function - | None -> Effect.ignore - | Some message -> - set_state (fun state -> - { state with import_completion = Some (operation, Some message) }))) - batch.batch_items)) + Queue.add (batch.batch_generation, target, staged) capture_import_queue) + batch.batch_items; + drain_imports ()) in let prev_upload_key = ref (upload_context initial_state) in let upload_callback () = @@ -5153,6 +5359,11 @@ let start | Capture_asset value -> (match value with | Journal_asset_import.Dismissed -> Effect.ignore + | Journal_asset_import.Picker_dismissed request_id -> + update (fun state -> + if request_id = Some state.capture_pick_request + then { state with capture_picker_armed = false } + else state) | Journal_asset_import.Unavailable message -> update (fun state -> { state with capture_error = Some (Local_capture_failure message) }) @@ -5172,6 +5383,9 @@ let start (fun capture -> Journal_capture.remove_attachment capture ~token) state.direct_capture }) + | Journal_asset_import.Picked_batch (items, request_id, error) -> + update (fun state -> + Root_navigation.step state (Capture_assets_picked (items, request_id, error))) | Journal_asset_import.Picked (staged, request_id) -> update (fun state -> Root_navigation.step state (Capture_asset_picked (staged, request_id)))) @@ -5228,7 +5442,7 @@ let start | Dismiss_operation_error -> update (fun state -> match state.timeline_notice with - | Some (Delete_failed _ | Status_failed _ | Copy_failed _) -> + | Some (Delete_failed _ | Status_failed _ | Copy_failed _ | Import_failed _) -> { state with timeline_notice = None } | None | Some Delete_undo -> state) | Close_error_info -> update (fun state -> { state with modal = No_modal }) @@ -5710,6 +5924,7 @@ let start && left.graph_state.generation = right.graph_state.generation && left.write_enabled = right.write_enabled && left.capture_error == right.capture_error + && left.capture_picker_armed = right.capture_picker_armed && left.capture_pick_request = right.capture_pick_request && left.capture_pick_source = right.capture_pick_source && left.import_completion = right.import_completion @@ -5821,8 +6036,16 @@ let start let capture_assets state ~camera = { request = Journal_asset_import.staged_request - ~id:state.capture_pick_request + ~id:(if state.capture_picker_armed then state.capture_pick_request else 0) ~source:state.capture_pick_source + ~max_selections: + (Journal_capture.attachment_limit + - Option.fold + ~none:0 + ~some:(fun capture -> + List.length (Journal_capture.pending_attachments capture)) + state.direct_capture) + () ; camera ; completion = state.import_completion ; on_attach = (fun source -> dispatch.send (Capture_attach source)) @@ -5984,7 +6207,7 @@ let start ~enabled:(current.write_enabled && Journal_capture.can_attach capture) ~completion:assets.completion ~request:assets.request - ~pending:[] + ~pending:(Journal_capture.pending_attachments capture) ~on_select:assets.on_event (V.Body.static (V.empty ()))) in @@ -6320,24 +6543,8 @@ let start model_signal ] in - let os = - match platform_code with - | 1 -> Lui_protocol.MacOS - | 2 -> Lui_protocol.IOS - | 3 -> Lui_protocol.AndroidOS - | 4 -> Lui_protocol.LinuxOS - | 5 -> Lui_protocol.WindowsOS - | _ -> Lui_protocol.GenericOS - in - let host = - match host_code with - | 1 -> Lui_protocol.WebHost - | 2 -> Lui_protocol.SwiftUIHost - | 3 -> Lui_protocol.FlutterHost - | _ -> Lui_protocol.GenericHost - in let backend = - { Lui_protocol.backend_profile = Lui_protocol.profile os host + { Lui_protocol.backend_profile = Lui_native_bridge.profile platform_code host_code ; apply_batch = (fun batch -> latest_patch := Lui_wire.encode_batch batch; @@ -6353,7 +6560,20 @@ let start view in app_cell := Some app; - let context = { app; pump; client; send_action; apply_platform; running } in + let release_staging () = + ignore (clear_account_drafts !state_ref); + Queue.iter + (fun (_, _, staged) -> Journal_asset_import.discard_staged_file staged) + capture_import_queue; + Queue.clear capture_import_queue; + Hashtbl.iter + (fun _ staged -> Journal_asset_import.discard_staged_file staged) + capture_staged_items; + Hashtbl.clear capture_staged_items + in + let context = + { app; pump; client; send_action; apply_platform; running; release_staging } + in current_app := Some context; ignore (Worker.send client Graph_service.Get_graph_state : Worker.send_result); Worker.on_event client (fun event -> @@ -6514,6 +6734,7 @@ let create | Some context -> context.running := false; Worker.Private.request_stop context.client; + context.release_staging (); ignore (Lui_app.dispose context.app); current_app := None | None -> ()); diff --git a/app/application.mli b/app/application.mli index 2f54ed7..d688ba0 100644 --- a/app/application.mli +++ b/app/application.mli @@ -116,6 +116,8 @@ module Root_navigation : sig | Capture_discarded | Capture_picker_requested of Journal_asset_import.source | Capture_asset_picked of Journal_asset_import.staged * int option + | Capture_assets_picked of + Journal_asset_import.staged list * int option * string option | Capture_native_edit of Journal_view.Event.Payload.text_edit | Capture_task_intent of bool | Capture_edited of string diff --git a/app/journal_asset_import.ml b/app/journal_asset_import.ml index 86c63fe..fca476d 100644 --- a/app/journal_asset_import.ml +++ b/app/journal_asset_import.ml @@ -19,10 +19,14 @@ type request = { id : int ; source : source ; staged : bool + ; max_selections : int option } -let file_request ~id = { id; source = Files; staged = false } -let staged_request ~id ~source = { id; source; staged = true } +let file_request ~id = { id; source = Files; staged = false; max_selections = Some 1 } + +let staged_request ?max_selections ~id ~source () = + { id; source; staged = true; max_selections } +;; (* A picked asset held for a later import: the wire pick fields plus [token], which identifies the pending item for the extension's remove event. *) @@ -35,12 +39,14 @@ type staged = ; source_file : string ; title : string ; file_type : string + ; source_identity : string option } let staged_token (staged : staged) = staged.token let staged_path (staged : staged) = staged.source_file let staged_title (staged : staged) = staged.title let staged_type (staged : staged) = staged.file_type +let staged_source_identity (staged : staged) = staged.source_identity let parse payload = let ( let* ) = Result.bind in @@ -79,6 +85,10 @@ let parse payload = ; source_file ; title ; file_type + ; source_identity = + (match Yojson.Basic.Util.member "sourceIdentity" json with + | `String value -> Some value + | _ -> None) } , request_id ) with @@ -117,8 +127,10 @@ let to_import (staged : staged) ~target : Logseq_db_types.Asset_import.t = type event = | Picked of staged * int option + | Picked_batch of staged list * int option * string option | Removed of string | Dismissed + | Picker_dismissed of int option | Unavailable of string let decode_event payload = @@ -128,7 +140,35 @@ let decode_event payload = with | `Assoc fields -> (match List.assoc_opt "action" fields with - | Some (`String "dismissed") -> Ok Dismissed + | Some (`String "picked-batch") -> + let request_id = + match Yojson.Basic.Util.member "request" (`Assoc fields) with + | `Assoc request -> + (match List.assoc_opt "id" request with + | Some (`Int id) -> Some id + | _ -> None) + | _ -> None + in + let error = + match List.assoc_opt "error" fields with + | Some (`String message) -> Some message + | _ -> None + in + (match List.assoc_opt "items" fields with + | Some (`List items) -> + let rec decode reversed = function + | [] -> Ok (Picked_batch (List.rev reversed, request_id, error)) + | item :: rest -> + (match parse (Yojson.Basic.to_string item) with + | Ok (staged, _) -> decode (staged :: reversed) rest + | Error message -> Error message) + in + decode [] items + | _ -> Error "Invalid attachment batch") + | Some (`String "dismissed") -> + (match List.assoc_opt "request" fields with + | Some (`Int id) -> Ok (Picker_dismissed (Some id)) + | _ -> Ok Dismissed) | Some (`String "unavailable") -> Ok (Unavailable @@ -167,7 +207,7 @@ let discard_staged_file (staged : staged) = let is_dismissal payload = match decode_event payload with - | Ok Dismissed -> true + | Ok Dismissed | Ok (Picker_dismissed _) -> true | _ -> false ;; @@ -194,6 +234,10 @@ let view ~key ~enabled ~completion ~request ~pending ~on_select body = [ "id", `Int request.id ; "source", `String (source_to_string request.source) ; "staged", `Bool request.staged + ; ( "maxSelections" + , match request.max_selections with + | None -> `Null + | Some count -> `Int count ) ] ) ; ( "pending" , `List @@ -204,6 +248,10 @@ let view ~key ~enabled ~completion ~request ~pending ~on_select body = ; "path", `String item.source_file ; "title", `String item.title ; "type", `String item.file_type + ; ( "sourceIdentity" + , match item.source_identity with + | None -> `Null + | Some identity -> `String identity ) ]) pending) ) ]) diff --git a/app/journal_asset_import.mli b/app/journal_asset_import.mli index 2002f3d..897c453 100644 --- a/app/journal_asset_import.mli +++ b/app/journal_asset_import.mli @@ -3,13 +3,13 @@ type source = | Photos | Camera - (** Picker arm request: [source] selects the picker, [staged] makes the host copy the pick into a temp file so it survives until a later import. *) type request = { id : int ; source : source ; staged : bool + ; max_selections : int option } (** In-place import request (detail page): file picker, no staging copy. *) @@ -17,7 +17,7 @@ val file_request : id:int -> request (** Staged request for deferred import (composer): the host copies the pick into a temp file and reports it for the pending-attachment strip. *) -val staged_request : id:int -> source:source -> request +val staged_request : ?max_selections:int -> id:int -> source:source -> unit -> request (** A picked asset kept for a later attach-on-save import. *) type staged @@ -26,6 +26,7 @@ val staged_token : staged -> string val staged_path : staged -> string val staged_title : staged -> string val staged_type : staged -> string +val staged_source_identity : staged -> string option (** Build the worker import for a staged pick targeting [target]. *) val to_import @@ -40,8 +41,10 @@ val decode type event = | Picked of staged * int option (** the pick and the request id that armed it *) + | Picked_batch of staged list * int option * string option | Removed of string | Dismissed + | Picker_dismissed of int option | Unavailable of string val decode_event : string -> (event, string) result diff --git a/app/journal_asset_policy.ml b/app/journal_asset_policy.ml index 82301da..90c193d 100644 --- a/app/journal_asset_policy.ml +++ b/app/journal_asset_policy.ml @@ -78,7 +78,10 @@ type event = ; today : int ; settings : settings } - | Graph_changed + | Resync + | Roots_changed of Graph.Uuid.t list + | Index_changed of reason + | Dependencies_unavailable | Roots_loaded of ticket * Graph.Uuid.t list * Graph.Cursor.t option | Assets_loaded of ticket * Asset.t list * Graph.Cursor.t option | Read_failed of ticket @@ -119,15 +122,41 @@ type phase = | Finished | Broken +module Uuids = Set.Make (struct + type t = Graph.Uuid.t + + let compare = Graph.Uuid.compare + end) + +module Batches = Map.Make (String) +module Names = Set.Make (String) + +type batch = + { roots : Graph.Uuid.t list + ; consumers : Names.t + } + +type mode = + | Full + | Index of Uuids.t + | Partial of query Rrbvec.t * int + type scan = { reason : reason ; enabled : bool ; dirty : bool ; root_query : query ; root_next : Graph.Cursor.t option - ; committed : string list - ; staged : string list + ; committed : Names.t + ; staged : Names.t ; phase : phase + ; batches : batch Batches.t + ; staged_batches : batch Batches.t + ; replacing : Names.t + ; mode : mode + ; changed_roots : Uuids.t + ; index_dirty : bool + ; known_roots : Uuids.t } type visible = @@ -140,7 +169,7 @@ type visible = module Consumers = Map.Make (String) module Versions = Map.Make (struct - type t = Graph.Uuid.t * Asset.source + type t = Graph.Uuid.t let compare = compare end) @@ -158,6 +187,7 @@ type t = ; scans : scan list ; visible : visible list ; residency : residency Versions.t Consumers.t + ; ready_owners : int Versions.t } let empty = @@ -168,6 +198,7 @@ let empty = ; scans = [] ; visible = [] ; residency = Consumers.empty + ; ready_owners = Versions.empty } ;; @@ -199,7 +230,9 @@ let demand (pending : pending_demand) = } ;; -let releases consumers = List.map (fun consumer -> Release consumer) consumers +let releases consumers = + Names.fold (fun consumer instructions -> Release consumer :: instructions) consumers [] +;; let with_scan state scan = { state with @@ -221,9 +254,42 @@ let issue state scan query = with_scan state { scan with phase = Reading ticket }, [ Read ticket ] ;; +let batch_key roots = String.concat ":" (List.map Graph.Uuid.to_string roots) +let root_set roots = List.fold_left (fun set root -> Uuids.add root set) Uuids.empty roots + +let batch_roots batches = + Batches.fold + (fun _ batch roots -> Uuids.union roots (root_set batch.roots)) + batches + Uuids.empty +;; + +let batch_consumers batches = + Batches.fold (fun _ batch all -> Names.union batch.consumers all) batches Names.empty +;; + let finish state scan = - ( with_scan state { scan with committed = scan.staged; staged = []; phase = Finished } - , releases scan.committed ) + let replaced, retained = + Batches.partition (fun key _ -> Names.mem key scan.replacing) scan.batches + in + let batches = + Batches.union (fun _ _ fresh -> Some fresh) retained scan.staged_batches + in + ( with_scan + state + { scan with + committed = batch_consumers batches + ; staged = Names.empty + ; batches + ; staged_batches = Batches.empty + ; replacing = Names.empty + ; phase = Finished + ; known_roots = + (match scan.mode with + | Full -> batch_roots batches + | Index _ | Partial _ -> scan.known_roots) + } + , releases (batch_consumers replaced) ) ;; let continue state scan = function @@ -232,13 +298,26 @@ let continue state scan = function ;; let root_continuation scan = - Option.map - (fun cursor -> - match scan.root_query with - | Recent_roots range -> Recent_roots { range with cursor = Some cursor } - | Favorite_roots _ -> Favorite_roots (Some cursor) - | Assets _ -> assert false) - scan.root_next + match scan.mode with + | Partial (queries, index) -> Rrbvec.nth_opt queries index + | Full | Index _ -> + Option.map + (fun cursor -> + match scan.root_query with + | Recent_roots range -> Recent_roots { range with cursor = Some cursor } + | Favorite_roots _ -> Favorite_roots (Some cursor) + | Assets _ -> assert false) + scan.root_next +;; + +let continue_roots state scan = + let query = root_continuation scan in + let scan = + match scan.mode with + | Partial (queries, index) -> { scan with mode = Partial (queries, index + 1) } + | _ -> scan + in + continue state scan query ;; let current state ticket = @@ -251,24 +330,166 @@ let current state ticket = ;; let fail state scan = - with_scan state { scan with staged = []; phase = Broken }, releases scan.staged + with_scan state { scan with staged = Names.empty; phase = Broken }, releases scan.staged ;; let scope_releases state = + let consumers = + List.fold_left + (fun consumers scan -> + Names.union consumers (Names.union scan.committed scan.staged)) + Names.empty + state.scans + in releases - (List.concat_map (fun scan -> scan.committed @ scan.staged) state.scans - @ List.map (fun (visible : visible) -> visible.consumer) state.visible) + (List.fold_left + (fun consumers (visible : visible) -> Names.add visible.consumer consumers) + consumers + state.visible) ;; let restart state scan = let cleanup = releases scan.staged in - let scan = { scan with dirty = false; root_next = None; staged = [] } in + let scan = + { scan with + dirty = false + ; root_next = None + ; staged = Names.empty + ; staged_batches = Batches.empty + ; replacing = + Batches.fold (fun key _ keys -> Names.add key keys) scan.batches Names.empty + ; mode = Full + ; changed_roots = Uuids.empty + ; index_dirty = false + } + in let state, instructions = continue state scan (if scan.enabled then Some scan.root_query else None) in state, cleanup @ instructions ;; +let partial state scan roots = + let selected = + Batches.filter + (fun _ batch -> not (Uuids.is_empty (Uuids.inter roots (root_set batch.roots)))) + scan.batches + in + if Batches.is_empty selected + then with_scan state { scan with changed_roots = Uuids.empty }, [] + else ( + let replacing = + Batches.fold (fun key _ keys -> Names.add key keys) selected Names.empty + in + let queries = + Batches.fold + (fun _ batch queries -> + Rrbvec.append_list queries [ Assets { roots = batch.roots; cursor = None } ]) + selected + Rrbvec.empty + in + let scan = + { scan with + staged = Names.empty + ; staged_batches = Batches.empty + ; replacing + ; mode = Partial (queries, 0) + ; changed_roots = Uuids.empty + ; root_next = None + } + in + continue_roots state scan) +;; + +let index state scan = + issue + state + { scan with + mode = Index Uuids.empty + ; root_next = None + ; index_dirty = false + ; staged = Names.empty + ; staged_batches = Batches.empty + ; replacing = Names.empty + } + scan.root_query +;; + +let finish_or_changes state scan = + let state, instructions = finish state scan in + let scan = List.find (fun other -> other.reason = scan.reason) state.scans in + let state, next = + if scan.index_dirty then index state scan else partial state scan scan.changed_roots + in + state, instructions @ next +;; + +let continue_scan state scan query = + match query with + | None -> finish_or_changes state scan + | Some query -> issue state scan query +;; + +let continue_scan_roots state scan = + let query = root_continuation scan in + let scan = + match scan.mode with + | Partial (queries, index) -> { scan with mode = Partial (queries, index + 1) } + | _ -> scan + in + continue_scan state scan query +;; + +let indexed state scan roots = + let old_roots = batch_roots scan.batches in + let removed = Uuids.diff old_roots roots in + let added = Uuids.diff roots old_roots in + let selected = + Batches.filter + (fun _ batch -> not (Uuids.is_empty (Uuids.inter removed (root_set batch.roots)))) + scan.batches + in + let replacing = + Batches.fold (fun key _ keys -> Names.add key keys) selected Names.empty + in + let kept_queries = + Batches.fold + (fun _ batch queries -> + let remaining = List.filter (fun root -> Uuids.mem root roots) batch.roots in + if remaining = [] + then queries + else Rrbvec.append_list queries [ Assets { roots = remaining; cursor = None } ]) + selected + Rrbvec.empty + in + let rec chunks remaining queries = + if Uuids.is_empty remaining + then queries + else ( + let batch, rest = + Uuids.fold + (fun root (batch, rest) -> + if List.length batch < page_size + then root :: batch, rest + else batch, Uuids.add root rest) + remaining + ([], Uuids.empty) + in + chunks + rest + (Rrbvec.append_list queries [ Assets { roots = List.rev batch; cursor = None } ])) + in + let scan = + { scan with + replacing + ; mode = Partial (Rrbvec.append kept_queries (chunks added Rrbvec.empty), 0) + ; root_next = None + ; known_roots = roots + } + in + continue_scan_roots state scan +;; + let step_policy state = function | Shutdown -> ( { empty with serial = state.serial; revision = state.revision + 1 } @@ -280,7 +501,12 @@ let step_policy state = function let prior = if same_scope then state.scans else [] in let cleanup = if same_scope - then releases (List.concat_map (fun scan -> scan.staged) prior) + then + releases + (List.fold_left + (fun consumers scan -> Names.union consumers scan.staged) + Names.empty + prior) else scope_releases state in let state = @@ -296,7 +522,7 @@ let step_policy state = function (fun (state, instructions) reason -> let committed = match List.find_opt (fun (scan : scan) -> scan.reason = reason) prior with - | None -> [] + | None -> Names.empty | Some scan -> scan.committed in let query = @@ -315,15 +541,35 @@ let step_policy state = function ; root_query = Option.value query ~default:(Favorite_roots None) ; root_next = None ; committed - ; staged = [] + ; staged = Names.empty ; phase = Finished + ; batches = + (match List.find_opt (fun scan -> scan.reason = reason) prior with + | None -> Batches.empty + | Some scan -> scan.batches) + ; staged_batches = Batches.empty + ; replacing = + (match List.find_opt (fun scan -> scan.reason = reason) prior with + | None -> Names.empty + | Some scan -> + Batches.fold + (fun key _ keys -> Names.add key keys) + scan.batches + Names.empty) + ; mode = Full + ; changed_roots = Uuids.empty + ; index_dirty = false + ; known_roots = + (match List.find_opt (fun scan -> scan.reason = reason) prior with + | None -> Uuids.empty + | Some scan -> scan.known_roots) } in let state, next = continue state scan query in state, instructions @ next) (state, cleanup) [ Recent; Favorites ] - | Graph_changed -> + | Resync -> let state = { state with revision = state.revision + 1 } in List.fold_left (fun (state, instructions) scan -> @@ -333,13 +579,52 @@ let step_policy state = function match scan.phase with | Reading _ | Awaiting _ -> let cleanup = releases scan.staged in - let scan = { scan with dirty = true; root_next = None; staged = [] } in + let scan = + { scan with dirty = true; root_next = None; staged = Names.empty } + in with_scan state scan, instructions @ cleanup | Pressured _ | Finished | Broken -> let state, next = restart state scan in state, instructions @ next)) (state, []) state.scans + | Roots_changed roots -> + let roots = root_set roots in + List.fold_left + (fun (state, instructions) scan -> + let owned = scan.known_roots in + let relevant = Uuids.inter roots owned in + if Uuids.is_empty relevant + then state, instructions + else ( + let scan = + { scan with changed_roots = Uuids.union scan.changed_roots relevant } + in + match scan.phase with + | Finished | Broken -> + let state, next = partial state scan scan.changed_roots in + state, instructions @ next + | Reading _ | Awaiting _ | Pressured _ -> with_scan state scan, instructions)) + (state, []) + state.scans + | Index_changed reason -> + (match + List.find_opt (fun scan -> scan.reason = reason && scan.enabled) state.scans + with + | None -> state, [] + | Some scan -> + (match scan.phase with + | Finished | Broken -> index state scan + | Reading _ | Awaiting _ | Pressured _ -> + with_scan state { scan with index_dirty = true }, [])) + | Dependencies_unavailable -> + ( { state with + scans = + List.map + (fun scan -> if scan.enabled then { scan with phase = Broken } else scan) + state.scans + } + , [] ) | Roots_loaded (ticket, roots, next_cursor) -> (match current state ticket with | None -> state, [] @@ -352,9 +637,29 @@ let step_policy state = function | Assets _ -> state, [] | Recent_roots _ | Favorite_roots _ -> let scan = { scan with root_next = next_cursor } in - if roots = [] - then continue state scan (root_continuation scan) - else issue state scan (Assets { roots; cursor = None }))) + (match scan.mode with + | Index collected -> + let collected = Uuids.union collected (root_set roots) in + let scan = { scan with mode = Index collected } in + (match next_cursor with + | None -> indexed state scan collected + | Some _ -> continue_scan_roots state scan) + | Full | Partial _ -> + if roots = [] + then continue_scan_roots state scan + else ( + let key = batch_key roots in + let scan = + { scan with + staged_batches = + Batches.add + key + { roots; consumers = Names.empty } + scan.staged_batches + ; known_roots = Uuids.union scan.known_roots (root_set roots) + } + in + issue state scan (Assets { roots; cursor = None }))))) | Assets_loaded (ticket, assets, next_cursor) -> (match current state ticket with | None -> state, [] @@ -372,8 +677,20 @@ let step_policy state = function | None -> root_continuation scan in let assets = managed assets in + let key = batch_key roots in + let existing = + Option.value + (Batches.find_opt key scan.staged_batches) + ~default:{ roots; consumers = Names.empty } + in + let scan = + { scan with staged_batches = Batches.add key existing scan.staged_batches } + in if assets = [] - then continue state scan continuation + then ( + match next_cursor with + | Some _ -> continue_scan state scan continuation + | None -> continue_scan_roots state scan) else ( let consumer = Printf.sprintf @@ -385,7 +702,15 @@ let step_policy state = function let pending = { consumer; assets; continuation } in ( with_scan state - { scan with staged = consumer :: scan.staged; phase = Awaiting pending } + { scan with + staged = Names.add consumer scan.staged + ; phase = Awaiting pending + ; staged_batches = + Batches.add + key + { existing with consumers = Names.add consumer existing.consumers } + scan.staged_batches + } , [ demand pending ] )))) | Read_failed ticket -> (match current state ticket with @@ -412,7 +737,11 @@ let step_policy state = function | Some scan when scan.dirty -> restart state scan | Some scan -> (match scan.phase with - | Awaiting p | Pressured p -> continue state scan p.continuation + | Awaiting p | Pressured p -> + (match p.continuation with + | Some (Assets { cursor = Some _; _ }) -> + continue_scan state scan p.continuation + | _ -> continue_scan_roots state scan) | _ -> assert false)) | Backpressure consumer -> let state = @@ -478,7 +807,37 @@ let step_policy state = function let selected, visible = List.partition (fun (v : visible) -> v.name = name) state.visible in - { state with visible }, releases (List.map (fun (v : visible) -> v.consumer) selected) + ( { state with visible } + , releases + (List.fold_left + (fun consumers (v : visible) -> Names.add v.consumer consumers) + Names.empty + selected) ) +;; + +let update_ready_owners ready_owners versions delta = + Versions.fold + (fun uuid resident counts -> + if not resident.ready + then counts + else + Versions.update + uuid + (fun current -> + let count = Option.value current ~default:0 + delta in + if count <= 0 then None else Some count) + counts) + versions + ready_owners +;; + +let replace_residency state consumer versions = + let previous = + Option.value (Consumers.find_opt consumer state.residency) ~default:Versions.empty + in + let ready_owners = update_ready_owners state.ready_owners previous (-1) in + let ready_owners = update_ready_owners ready_owners versions 1 in + { state with residency = Consumers.add consumer versions state.residency; ready_owners } ;; let register state consumer assets = @@ -486,49 +845,43 @@ let register state consumer assets = Option.value (Consumers.find_opt consumer state.residency) ~default:Versions.empty in let versions = - List.fold_left + Rrbvec.fold_left (fun versions (asset : Asset.t) -> match asset.source with | External _ -> versions | Managed _ -> - let key = asset.uuid, asset.source in - let residency = - Option.value - (Versions.find_opt key previous) - ~default:{ ready = false; failed = false } + let resident = + match Versions.find_opt asset.uuid previous with + | Some resident -> resident + | None -> + { ready = Versions.mem asset.uuid state.ready_owners; failed = false } in - Versions.add key residency versions) + Versions.add asset.uuid resident versions) Versions.empty - assets + (Rrbvec.of_list assets) in - { state with residency = Consumers.add consumer versions state.residency } + replace_residency state consumer versions ;; let step state event = let state = match event with | Availability { consumer; asset; availability } -> - let residency = - Consumers.update - consumer - (Option.map (fun versions -> - Versions.mapi - (fun (uuid, _) resident -> - if Graph.Uuid.equal uuid asset - then ( - match availability with - | Transfer.Ready _ -> { ready = true; failed = false } - | Failed _ -> { ready = false; failed = true } - | Queued - | Downloading - | Waiting_remote - | Waiting_network - | Waiting_unlock -> { ready = false; failed = false }) - else resident) - versions)) - state.residency - in - { state with residency } + (match Consumers.find_opt consumer state.residency with + | None -> state + | Some versions -> + let versions = + Versions.update + asset + (Option.map (fun _ -> + match availability with + | Transfer.Ready _ -> { ready = true; failed = false } + | Failed _ -> { ready = false; failed = true } + | Queued | Downloading | Waiting_remote | Waiting_network | Waiting_unlock + -> { ready = false; failed = false })) + versions + in + replace_residency state consumer versions) | _ -> state in let state, instructions = step_policy state event in @@ -537,7 +890,15 @@ let step state event = (fun state -> function | Demand { consumer; assets; _ } -> register state consumer assets | Release consumer -> - { state with residency = Consumers.remove consumer state.residency } + let previous = + Option.value + (Consumers.find_opt consumer state.residency) + ~default:Versions.empty + in + { state with + residency = Consumers.remove consumer state.residency + ; ready_owners = update_ready_owners state.ready_owners previous (-1) + } | Read _ -> state) state instructions @@ -555,15 +916,15 @@ type offline = let offline state reason = let consumers = match List.find_opt (fun (scan : scan) -> scan.reason = reason) state.scans with - | None -> [] + | None -> Names.empty | Some scan -> (match scan.phase with | Finished -> scan.committed - | _ -> scan.staged) + | _ -> Names.union scan.committed scan.staged) in let versions = - List.fold_left - (fun versions consumer -> + Names.fold + (fun consumer versions -> match Consumers.find_opt consumer state.residency with | None -> versions | Some current -> @@ -572,8 +933,8 @@ let offline state reason = Some { ready = a.ready && b.ready; failed = a.failed || b.failed }) versions current) - Versions.empty consumers + Versions.empty in Versions.fold (fun _ (resident : residency) status -> @@ -585,3 +946,9 @@ let offline state reason = versions { enumeration = progress state reason; total = 0; ready = 0; failed = 0 } ;; + +let roots state reason = + match List.find_opt (fun scan -> scan.reason = reason) state.scans with + | None -> Seq.empty + | Some scan -> Uuids.to_seq scan.known_roots +;; diff --git a/app/journal_asset_policy.mli b/app/journal_asset_policy.mli index 2b65c77..0891da7 100644 --- a/app/journal_asset_policy.mli +++ b/app/journal_asset_policy.mli @@ -46,10 +46,11 @@ type event = { graph_generation : int ; today : int ; settings : settings - } - (** A projection change or resync invalidates both configured scans. In-flight - reads finish before one fresh scan starts; their results are discarded. *) - | Graph_changed + } (** Only an explicit resync invalidates every configured scan. *) + | Resync + | Roots_changed of Graph.Uuid.t list + | Index_changed of reason + | Dependencies_unavailable | Roots_loaded of ticket * Graph.Uuid.t list * Graph.Cursor.t option | Assets_loaded of ticket * Asset.t list * Graph.Cursor.t option | Read_failed of ticket @@ -81,6 +82,7 @@ val empty : t val progress : t -> reason -> progress val step : t -> event -> t * instruction list val page_size : int +val roots : t -> reason -> Graph.Uuid.t Seq.t type offline = { enumeration : progress diff --git a/app/journal_asset_runtime.ml b/app/journal_asset_runtime.ml index 96e23a0..fd0ffdb 100644 --- a/app/journal_asset_runtime.ml +++ b/app/journal_asset_runtime.ml @@ -3,6 +3,37 @@ module Protocol = Logseq_db_worker.Protocol module Graph = Logseq_db_types.Graph_types module Service = Logseq_db_worker_lui.Logseq_db_worker_lui_service +module Uuids = Set.Make (struct + type t = Graph.Uuid.t + + let compare = Graph.Uuid.compare + end) + +module Properties = Map.Make (struct + type t = Graph.Uuid.t + + let compare = Graph.Uuid.compare + end) + +(* These are session resource bounds, independent of the total graph size. + Exhaustion publishes Failed without releasing resident consumers or issuing + a full-scope query. A genuine resync or graph replacement clears the cache. *) +let maximum_dependency_facts = 4096 +let maximum_accepted_fact_requests = 4096 +let maximum_queued_dependency_reads = 4096 +let maximum_active_dependency_reads = 4 +let maximum_ancestry_depth = 256 + +type fact = + { block : Graph.block option + ; issuance : int + ; references : Uuids.t + } + +type dependency = + | Block of Graph.Uuid.t + | Page of Graph.Uuid.t + type t = { send : Service.request -> bool ; changed : int option -> Policy.offline -> Policy.offline -> unit @@ -10,8 +41,22 @@ type t = ; mutable policy : Policy.t ; mutable configuration : (int * int * Policy.settings) option ; mutable generation : int option - ; mutable queued : Policy.instruction list + ; mutable queued : Policy.instruction Queue.t ; pending : (string, Policy.ticket) Hashtbl.t + ; facts : (string, fact) Hashtbl.t + ; pages : (string, Graph.page option * int) Hashtbl.t + ; page_dirty : (string, Graph.page option * Uuids.t * int * bool) Hashtbl.t + ; assets : (string, unit) Hashtbl.t + ; requests : (string, int * Protocol.command) Hashtbl.t + ; dependencies : (string, dependency * int) Hashtbl.t + ; dependency_queue : dependency Queue.t + ; dependency_waiting : (string, unit) Hashtbl.t + ; mutable serial : int + ; dirty : (string, Graph.block option * Uuids.t * int * bool) Hashtbl.t + ; mutable barrier : int + ; mutable degraded : bool + ; mutable favorites_page : Graph.Uuid.t option + ; mutable favorite_members : Uuids.t } let create ~send ~changed = @@ -21,8 +66,22 @@ let create ~send ~changed = ; policy = Policy.empty ; configuration = None ; generation = None - ; queued = [] + ; queued = Queue.create () ; pending = Hashtbl.create 2 + ; facts = Hashtbl.create 32 + ; pages = Hashtbl.create 8 + ; page_dirty = Hashtbl.create 8 + ; assets = Hashtbl.create 32 + ; requests = Hashtbl.create 32 + ; dependencies = Hashtbl.create 4 + ; dependency_queue = Queue.create () + ; dependency_waiting = Hashtbl.create 32 + ; serial = 0 + ; dirty = Hashtbl.create 32 + ; barrier = 0 + ; degraded = false + ; favorites_page = None + ; favorite_members = Uuids.empty } ;; @@ -36,6 +95,44 @@ let command (ticket : Policy.ticket) = V2_list_assets { recursive = true; roots; cursor; limit = Policy.page_size } ;; +let degrade t = + if not t.degraded + then ( + t.degraded <- true; + let policy, _ = Policy.step t.policy Dependencies_unavailable in + t.policy <- policy; + let recent = Policy.offline t.policy Recent + and favorites = Policy.offline t.policy Favorites in + t.changed t.generation recent favorites) +;; + +let observe_request t (request : Protocol.request) = + let key = Graph.Uuid.to_string request.request_id in + if not (Hashtbl.mem t.requests key) + then ( + if Hashtbl.length t.requests >= maximum_accepted_fact_requests + then ( + let oldest = + Hashtbl.fold + (fun key (issuance, _) candidate -> + if Hashtbl.mem t.pending key || Hashtbl.mem t.dependencies key + then candidate + else ( + match candidate with + | None -> Some (key, issuance) + | Some (_, prior) when issuance < prior -> Some (key, issuance) + | _ -> candidate)) + t.requests + None + in + Option.iter (fun (key, _) -> Hashtbl.remove t.requests key) oldest; + degrade t); + if Hashtbl.length t.requests < maximum_accepted_fact_requests + then ( + t.serial <- t.serial + 1; + Hashtbl.replace t.requests key (t.serial, request.command))) +;; + let execute t = function | Policy.Read ticket -> let request_id = @@ -46,6 +143,7 @@ let execute t = function t.send (Service.Graph_request { api_version = 2; request_id; command = command ticket }) then ( + observe_request t Protocol.{ api_version = 2; request_id; command = command ticket }; Hashtbl.replace t.pending (Graph.Uuid.to_string request_id) ticket; true) else false @@ -67,14 +165,79 @@ let execute t = function { graph_generation; command = Release_asset_demand consumer })) ;; -let rec pump t = - match t.queued with - | [] -> () - | instruction :: rest -> - if execute t instruction +let forget_request t ~request_id = + Hashtbl.remove t.requests (Graph.Uuid.to_string request_id) +;; + +let dependency_key = function + | Block uuid -> "b:" ^ Graph.Uuid.to_string uuid + | Page uuid -> "p:" ^ Graph.Uuid.to_string uuid +;; + +let enqueue t dependency = + let key = dependency_key dependency in + if + Option.is_some t.configuration + && (not t.degraded) + && (not (Hashtbl.mem t.dependency_waiting key)) + && not + (Hashtbl.fold + (fun _ (item, _) found -> found || dependency_key item = key) + t.dependencies + false) + then + if Queue.length t.dependency_queue >= maximum_queued_dependency_reads + then degrade t + else ( + Queue.add dependency t.dependency_queue; + Hashtbl.replace t.dependency_waiting key ()) +;; + +let rec pump_dependencies t = + if (not t.degraded) && Hashtbl.length t.dependencies < maximum_active_dependency_reads + then + if not (Queue.is_empty t.dependency_queue) then ( - t.queued <- rest; - pump t) + let dependency = Queue.peek t.dependency_queue in + let request_id = + Graph.Uuid.of_string (Printf.sprintf "a55e7100-0000-4000-8000-%012x" t.serial) + |> Result.get_ok + in + let command = + match dependency with + | Block block -> Protocol.V2_get_block { block; revision = None } + | Page page -> V2_get_page { page; revision = None } + in + let request = Protocol.{ api_version = 2; request_id; command } in + if t.send (Service.Graph_request request) + then ( + observe_request t request; + Hashtbl.replace + t.dependencies + (Graph.Uuid.to_string request_id) + (dependency, t.serial); + ignore (Queue.take t.dependency_queue : dependency); + Hashtbl.remove t.dependency_waiting (dependency_key dependency); + pump_dependencies t)) +;; + +let rec pump t = + if Queue.is_empty t.queued + then pump_dependencies t + else if execute t (Queue.peek t.queued) + then ( + ignore (Queue.take t.queued : Policy.instruction); + pump t) +;; + +let retain_releases t = + let retained = Queue.create () in + Queue.iter + (function + | Policy.Release _ as instruction -> Queue.add instruction retained + | Read _ | Demand _ -> ()) + t.queued; + t.queued <- retained ;; let publish t = @@ -90,7 +253,7 @@ let publish t = let dispatch t event = let policy, instructions = Policy.step t.policy event in t.policy <- policy; - t.queued <- t.queued @ instructions; + List.iter (fun instruction -> Queue.add instruction t.queued) instructions; pump t; publish t ;; @@ -100,40 +263,531 @@ let refresh t ~graph_generation ~today ~settings = if t.configuration <> Some configuration then ( Hashtbl.clear t.pending; - t.queued - <- List.filter - (function - | Policy.Release _ -> true - | Read _ | Demand _ -> false) - t.queued; + if t.generation <> Some graph_generation + then ( + Hashtbl.clear t.facts; + Hashtbl.clear t.pages; + Hashtbl.clear t.page_dirty; + Hashtbl.clear t.assets; + Hashtbl.clear t.dirty; + Hashtbl.clear t.requests; + Hashtbl.clear t.dependencies; + Queue.clear t.dependency_queue; + Hashtbl.clear t.dependency_waiting; + t.favorites_page <- None; + t.favorite_members <- Uuids.empty; + t.degraded <- false); + retain_releases t; t.configuration <- Some configuration; t.generation <- Some graph_generation); dispatch t (Refresh { graph_generation; today; settings }) ;; -let invalidate t = dispatch t Graph_changed +let resync t = + Hashtbl.clear t.dirty; + Hashtbl.clear t.facts; + Hashtbl.clear t.pages; + Hashtbl.clear t.page_dirty; + Hashtbl.clear t.assets; + Queue.clear t.dependency_queue; + Hashtbl.clear t.dependency_waiting; + t.degraded <- false; + t.barrier <- t.serial; + dispatch t Resync +;; + +let all_roots t = + Seq.append (Policy.roots t.policy Recent) (Policy.roots t.policy Favorites) +;; + +let fact t uuid = + Option.bind + (Hashtbl.find_opt t.facts (Graph.Uuid.to_string uuid)) + (fun fact -> fact.block) +;; + +let fold_values f state values = Rrbvec.fold_left f state (Rrbvec.of_list values) + +let rec value_uuids set = function + | Graph.Internal_uuid uuid -> Uuids.add uuid set + | Internal_list values -> fold_values value_uuids set values + | Internal_map values -> + fold_values + (fun set (key, value) -> value_uuids (value_uuids set key) value) + set + values + | Internal_null + | Internal_bool _ + | Internal_number _ + | Internal_string _ + | Internal_keyword _ -> set +;; + +let property_value_uuids set = function + | Graph.Node_value uuid + | Asset_value uuid + | Entity_value uuid + | Class_value uuid + | Page_value uuid -> Uuids.add uuid set + | Collection_value values -> fold_values value_uuids set values + | Map_value values -> + fold_values + (fun set (key, value) -> value_uuids (value_uuids set key) value) + set + values + | Any_value value -> value_uuids set value + | _ -> set +;; + +let uuid_set values = fold_values (fun set uuid -> Uuids.add uuid set) Uuids.empty values + +let property_membership properties = + fold_values + (fun members (property : Graph.property_summary) -> + Properties.add + property.uuid + ( property.schema + , property.values_truncated + , fold_values property_value_uuids Uuids.empty property.values ) + members) + Properties.empty + properties +;; + +let references (block : Graph.block) = + fold_values + (fun set (property : Graph.property_summary) -> + fold_values property_value_uuids (Uuids.add property.uuid set) property.values) + (uuid_set block.refs) + block.properties +;; + +let path ?(discover = true) t start = + let rec walk seen depth uuid = + if depth = maximum_ancestry_depth || Uuids.mem uuid seen + then seen + else ( + let seen = Uuids.add uuid seen in + match fact t uuid with + | None -> + if + discover + && (not (Hashtbl.mem t.pages (Graph.Uuid.to_string uuid))) + && not (Hashtbl.mem t.facts (Graph.Uuid.to_string uuid)) + then enqueue t (Block uuid); + seen + | Some block -> walk (Uuids.add block.page seen) (depth + 1) block.parent) + in + walk Uuids.empty 0 start +;; + +let roots_for_block t (block : Graph.block) = + let parents = + if Graph.Uuid.equal block.parent block.page then Uuids.empty else path t block.parent + in + let ancestors = Uuids.add block.uuid (Uuids.add block.page parents) in + Seq.fold_left + (fun roots root -> if Uuids.mem root ancestors then Uuids.add root roots else roots) + Uuids.empty + (all_roots t) +;; + +let roots_for_uuid t uuid = + let roots = + Seq.fold_left + (fun roots root -> + if Uuids.mem uuid (path ~discover:false t root) + then Uuids.add root roots + else roots) + Uuids.empty + (all_roots t) + in + Hashtbl.fold + (fun _ fact roots -> + match fact.block with + | Some block + when Graph.Uuid.equal block.uuid uuid || Uuids.mem uuid fact.references -> + Uuids.union roots (roots_for_block t block) + | None | Some _ -> roots) + t.facts + roots +;; + +let property_membership_equal = + Properties.equal + (fun (schema, truncated, values) (other, other_truncated, other_values) -> + schema = other + && (not truncated) + && (not other_truncated) + && Uuids.equal values other_values) +;; + +let block_membership_equal = + Option.equal + (fun + (parent, page, refs, tags, properties) + (other_parent, other_page, other_refs, other_tags, other_properties) + -> + Graph.Uuid.equal parent other_parent + && Graph.Uuid.equal page other_page + && Uuids.equal refs other_refs + && Uuids.equal tags other_tags + && property_membership_equal properties other_properties) +;; + +let page_membership_equal = + Option.equal + (fun (recycled, tags, properties) (other_recycled, other_tags, other_properties) -> + recycled = other_recycled + && Uuids.equal tags other_tags + && property_membership_equal properties other_properties) +;; + +let signature = + Option.map (fun (block : Graph.block) -> + ( block.parent + , block.page + , uuid_set block.refs + , uuid_set block.tags + , property_membership block.properties )) +;; + +let settle_dirty t = + let affected = ref Uuids.empty in + let complete = ref [] in + Hashtbl.iter + (fun key (before, old_roots, barrier, observed) -> + match Hashtbl.find_opt t.facts key with + | None -> () + | Some current when current.issuance > barrier -> + if + (not (block_membership_equal (signature before) (signature current.block))) + || ((not observed) && not (Uuids.is_empty old_roots)) + then ( + let roots = + match current.block with + | None -> Uuids.empty + | Some block -> roots_for_block t block + in + affected := Uuids.union !affected (Uuids.union old_roots roots)); + (* A known ancestry chain completes here. Unknown ancestors remain pending. *) + let resolved = + match current.block with + | None -> true + | Some block when Graph.Uuid.equal block.parent block.page -> true + | Some block -> + let ancestors = path t block.parent in + Uuids.for_all + (fun uuid -> + Hashtbl.mem t.facts (Graph.Uuid.to_string uuid) + || Hashtbl.mem t.pages (Graph.Uuid.to_string uuid)) + ancestors + in + if resolved then complete := key :: !complete + | Some _ -> ()) + t.dirty; + List.iter (Hashtbl.remove t.dirty) !complete; + if not (Uuids.is_empty !affected) + then dispatch t (Roots_changed (Uuids.elements !affected)); + pump t +;; + +let observe_response t (Protocol.V2_response { request_id; outcome; _ }) = + match Hashtbl.find_opt t.requests (Graph.Uuid.to_string request_id) with + | None -> () + | Some (issuance, command) -> + Hashtbl.remove t.requests (Graph.Uuid.to_string request_id); + if issuance > t.barrier + then ( + let remember_block uuid block = + let key = Graph.Uuid.to_string uuid in + let prior = Hashtbl.find_opt t.facts key in + if + (Hashtbl.length t.facts < maximum_dependency_facts || Option.is_some prior) + && + match prior with + | None -> true + | Some fact -> issuance >= fact.issuance + then ( + Hashtbl.replace + t.facts + key + { block + ; issuance + ; references = Option.fold ~none:Uuids.empty ~some:references block + }; + if Seq.exists (Graph.Uuid.equal uuid) (all_roots t) + then + Option.iter + (fun (block : Graph.block) -> ignore (path t block.parent : Uuids.t)) + block) + else if Option.is_none prior + then degrade t + in + let remember_lookup = function + | Protocol.V2_present_block { value; _ } -> + remember_block value.block.uuid (Some value.block) + | V2_missing_block { uuid; _ } -> remember_block uuid None + in + let remember_page_value uuid page = + let key = Graph.Uuid.to_string uuid in + let prior = Hashtbl.find_opt t.pages key in + if Hashtbl.length t.pages < maximum_dependency_facts || Option.is_some prior + then ( + if + match prior with + | None -> true + | Some (_, stamp) -> issuance >= stamp + then ( + Hashtbl.replace t.pages key (page, issuance); + match Hashtbl.find_opt t.page_dirty key with + | Some (before, roots, barrier, observed) when issuance > barrier -> + let direct = + Seq.exists (Graph.Uuid.equal uuid) (all_roots t) + || Hashtbl.fold + (fun _ fact found -> found || Uuids.mem uuid fact.references) + t.facts + false + in + let changed = + if direct + then ( + let membership = + Option.map (fun (page : Graph.page) -> + ( page.recycled + , uuid_set page.tags + , property_membership page.properties )) + in + (not (page_membership_equal (membership before) (membership page))) + || not observed) + else ( + let live = function + | Some (page : Graph.page) -> not page.recycled + | None -> false + in + (if observed then live before else true) <> live page) + in + if changed then dispatch t (Roots_changed (Uuids.elements roots)); + Hashtbl.remove t.page_dirty key + | _ -> ())) + else degrade t + in + let remember_page = function + | Protocol.V2_present_page { page; _ } -> + remember_page_value page.uuid (Some page) + | V2_missing_page { uuid; _ } -> remember_page_value uuid None + in + (match outcome with + | V2_block_outcome lookup -> remember_lookup lookup + | V2_page_outcome lookup -> + remember_page lookup; + (match lookup with + | V2_present_page { page; _ } + when page.name = "$$$favorites" && t.favorites_page <> Some page.uuid -> + dispatch t (Index_changed Favorites) + | _ -> ()) + | V2_block_summary_outcome { lookup; page; items; _ } -> + remember_lookup lookup; + Option.iter remember_page page; + List.iter + (fun (item : Protocol.v2_child_member) -> + remember_block item.value.block.uuid (Some item.value.block)) + items + | V2_children_outcome { items; _ } -> + List.iter + (fun (item : Protocol.v2_child_member) -> + remember_block item.value.block.uuid (Some item.value.block)) + items + | V2_page_tree_outcome { items; _ } -> + List.iter + (fun (item : Protocol.v2_tree_member) -> + remember_block item.value.block.uuid (Some item.value.block)) + items + | V2_journals_outcome { items; _ } -> + List.iter + (fun (item : Protocol.v2_journal_item) -> + remember_page_value item.page.uuid (Some item.page)) + items + | V2_favorites_outcome { favorites_page; items; _ } -> + t.favorites_page <- favorites_page; + (match command with + | V2_list_favorites { cursor = None; _ } -> t.favorite_members <- Uuids.empty + | _ -> ()); + List.iter + (fun (item : Protocol.v2_favorite_item) -> + t.favorite_members <- Uuids.add item.membership_uuid t.favorite_members; + match item.target with + | V2_favorite_block { uuid; _ } -> enqueue t (Block uuid) + | V2_favorite_page { uuid; _ } -> + if not (Hashtbl.mem t.pages (Graph.Uuid.to_string uuid)) + then enqueue t (Page uuid)) + items + | V2_assets_outcome { items; _ } -> + List.iter + (fun (asset : Logseq_db_types.Asset_descriptor.t) -> + if + Hashtbl.length t.assets < maximum_dependency_facts + || Hashtbl.mem t.assets (Graph.Uuid.to_string asset.uuid) + then Hashtbl.replace t.assets (Graph.Uuid.to_string asset.uuid) () + else degrade t) + items + | _ -> ()); + settle_dirty t) +;; + +let changes t windows = + if Option.is_some t.configuration && not t.degraded + then ( + let affected = ref Uuids.empty in + List.iter + (fun (window : Protocol.v2_change_window) -> + List.iter + (function + | Protocol.V2_journal_index_interest -> dispatch t (Index_changed Recent) + | V2_children_interest uuid | V2_page_tree_interest uuid -> + let direct = + Seq.fold_left + (fun roots root -> + if Graph.Uuid.equal root uuid then Uuids.add root roots else roots) + Uuids.empty + (all_roots t) + in + let below = + match fact t uuid with + | None -> Uuids.empty + | Some block -> roots_for_block t block + in + affected := Uuids.union !affected (Uuids.union direct below); + if t.favorites_page = Some uuid then dispatch t (Index_changed Favorites); + if + Option.is_none (fact t uuid) + && (not (Hashtbl.mem t.facts (Graph.Uuid.to_string uuid))) + && (not + (List.exists + (function + | Protocol.V2_page_tree_interest page -> + Graph.Uuid.equal page uuid + | _ -> false) + window.structure_interests)) + && not (Hashtbl.mem t.pages (Graph.Uuid.to_string uuid)) + then ( + let key = Graph.Uuid.to_string uuid in + if not (Hashtbl.mem t.dirty key) + then + if Hashtbl.length t.dirty >= maximum_dependency_facts + then degrade t + else Hashtbl.replace t.dirty key (None, Uuids.empty, t.serial, false); + enqueue t (Block uuid))) + window.structure_interests; + List.iter + (fun uuid -> + if t.favorites_page = Some uuid || Uuids.mem uuid t.favorite_members + then dispatch t (Index_changed Favorites); + if not (Hashtbl.mem t.assets (Graph.Uuid.to_string uuid)) + then ( + let key = Graph.Uuid.to_string uuid in + (match Hashtbl.find_opt t.dirty key with + | Some (before, roots, _, observed) -> + Hashtbl.replace t.dirty key (before, roots, t.serial, observed) + | None -> + if Hashtbl.length t.dirty >= maximum_dependency_facts + then degrade t + else + Hashtbl.replace + t.dirty + key + ( fact t uuid + , roots_for_uuid t uuid + , t.serial + , Hashtbl.mem t.facts key )); + enqueue t (Block uuid))) + window.block_uuids; + List.iter + (fun uuid -> + let roots = roots_for_uuid t uuid in + if not (Uuids.is_empty roots) + then ( + let key = Graph.Uuid.to_string uuid in + let before = Hashtbl.find_opt t.pages key in + if + Hashtbl.length t.page_dirty < maximum_dependency_facts + || Hashtbl.mem t.page_dirty key + then ( + (match Hashtbl.find_opt t.page_dirty key with + | Some (before, roots, _, observed) -> + Hashtbl.replace t.page_dirty key (before, roots, t.serial, observed) + | None -> + Hashtbl.replace + t.page_dirty + key + (Option.bind before fst, roots, t.serial, Option.is_some before)); + enqueue t (Page uuid)) + else degrade t) + else if t.favorites_page = None + then enqueue t (Page uuid)) + window.page_uuids) + windows; + if not (Uuids.is_empty !affected) + then dispatch t (Roots_changed (Uuids.elements !affected)); + pump t) +;; let shutdown t = - t.queued - <- List.filter - (function - | Policy.Release _ -> true - | Read _ | Demand _ -> false) - t.queued; + retain_releases t; dispatch t Shutdown; Hashtbl.clear t.pending; - t.queued <- []; + Hashtbl.clear t.facts; + Hashtbl.clear t.pages; + Hashtbl.clear t.page_dirty; + Hashtbl.clear t.assets; + Hashtbl.clear t.dirty; + Hashtbl.clear t.requests; + Hashtbl.clear t.dependencies; + Queue.clear t.dependency_queue; + Hashtbl.clear t.dependency_waiting; + t.favorites_page <- None; + t.favorite_members <- Uuids.empty; + Queue.clear t.queued; t.configuration <- None; t.generation <- None; publish t ;; -let receive t (Protocol.V2_response { request_id; outcome; _ }) = +let receive t (Protocol.V2_response { request_id; outcome; _ } as response) = let key = Graph.Uuid.to_string request_id in match Hashtbl.find_opt t.pending key with - | None -> false + | None -> + (match Hashtbl.find_opt t.dependencies key with + | None -> false + | Some (dependency, issuance) -> + Hashtbl.remove t.dependencies key; + let expected = + match dependency, outcome with + | Block _, Protocol.V2_block_outcome _ | Page _, V2_page_outcome _ -> true + | _ -> false + in + if not expected + then ( + Hashtbl.clear t.dirty; + Hashtbl.clear t.page_dirty; + Queue.clear t.dependency_queue; + Hashtbl.clear t.dependency_waiting; + degrade t); + observe_response t response; + (match dependency with + | Block uuid -> + (match Hashtbl.find_opt t.dirty (Graph.Uuid.to_string uuid) with + | Some (_, _, barrier, _) when issuance <= barrier -> enqueue t dependency + | _ -> ()) + | Page uuid -> + (match Hashtbl.find_opt t.page_dirty (Graph.Uuid.to_string uuid) with + | Some (_, _, barrier, _) when issuance <= barrier -> enqueue t dependency + | _ -> ())); + pump t; + true) | Some ticket -> Hashtbl.remove t.pending key; + observe_response t response; let event = match ticket.query, outcome with | Policy.Recent_roots _, Protocol.V2_journals_outcome { items; next_cursor } -> @@ -159,9 +813,20 @@ let receive t (Protocol.V2_response { request_id; outcome; _ }) = ;; let reject t ~request_id = + forget_request t ~request_id; let key = Graph.Uuid.to_string request_id in match Hashtbl.find_opt t.pending key with - | None -> () + | None -> + if Hashtbl.mem t.dependencies key + then ( + Hashtbl.clear t.dirty; + Hashtbl.clear t.page_dirty; + Queue.clear t.dependency_queue; + Hashtbl.clear t.dependency_waiting; + degrade t); + Hashtbl.remove t.dependencies key; + Hashtbl.remove t.requests key; + pump t | Some ticket -> Hashtbl.remove t.pending key; dispatch t (Read_failed ticket) diff --git a/app/journal_asset_runtime.mli b/app/journal_asset_runtime.mli index 852dff1..cebb07c 100644 --- a/app/journal_asset_runtime.mli +++ b/app/journal_asset_runtime.mli @@ -18,10 +18,19 @@ val refresh -> settings:Journal_asset_policy.settings -> unit -(** Invalidate configured scans after a projection change or resync. Unknown - dependencies conservatively affect both scans; global projection cursors - are discarded. Pending requests coalesce and retain their terminal owner. *) -val invalidate : t -> unit +(** Observe only accepted, currently owned requests/responses. Issuance fences + late responses; facts are discarded on graph replacement. *) +val observe_request : t -> Logseq_db_worker.Protocol.request -> unit + +val forget_request : t -> request_id:Logseq_db_types.Graph_types.Uuid.t -> unit +val observe_response : t -> Logseq_db_worker.Protocol.response -> unit + +(** Reconcile only dependency-intersecting bounded recursive root batches. + Unknown holders use bounded point reads instead of invalidating all scans. *) +val changes : t -> Logseq_db_worker.Protocol.v2_change_window list -> unit + +(** Explicit projection resync restarts the configured index enumerations. *) +val resync : t -> unit val shutdown : t -> unit val receive : t -> Logseq_db_worker.Protocol.response -> bool diff --git a/app/journal_bridge.ml b/app/journal_bridge.ml index d978e82..6786cf4 100644 --- a/app/journal_bridge.ml +++ b/app/journal_bridge.ml @@ -29,6 +29,30 @@ let dispatch_lui event = (hooks ()).dispatch event let appear node = dispatch_lui (Lui_protocol.Appear node) let press node = dispatch_lui (Lui_protocol.Press node) let long_press node = dispatch_lui (Lui_protocol.LongPress node) + +let pointer_detail x y modifiers button target_class : Lui_protocol.pointer_detail = + { x; y; modifiers; button; target_class } +;; + +let press_detail node x y modifiers button target_class = + dispatch_lui (Lui_protocol.PressDetail (node, pointer_detail x y modifiers button target_class)) +;; + +let pointer_down node x y modifiers button target_class = + dispatch_lui (Lui_protocol.PointerDown (node, pointer_detail x y modifiers button target_class)) +;; + +let pointer_up node x y modifiers button target_class = + dispatch_lui (Lui_protocol.PointerUp (node, pointer_detail x y modifiers button target_class)) +;; + +let context_menu_press node x y modifiers button target_class = + dispatch_lui (Lui_protocol.ContextMenuPress (node, pointer_detail x y modifiers button target_class)) +;; + +let pointer_enter node = dispatch_lui (Lui_protocol.PointerEnter node) +let pointer_leave node = dispatch_lui (Lui_protocol.PointerLeave node) + let text_changed node text = dispatch_lui (Lui_protocol.TextChanged (node, text)) let submit node = dispatch_lui (Lui_protocol.Submit node) let dismiss node = dispatch_lui (Lui_protocol.Dismiss node) @@ -63,6 +87,12 @@ let register hooks = Callback.register "lui_ocaml_appear" appear; Callback.register "lui_ocaml_press" press; Callback.register "lui_ocaml_long_press" long_press; + Callback.register "lui_ocaml_press_detail" press_detail; + Callback.register "lui_ocaml_pointer_down" pointer_down; + Callback.register "lui_ocaml_pointer_up" pointer_up; + Callback.register "lui_ocaml_pointer_enter" pointer_enter; + Callback.register "lui_ocaml_pointer_leave" pointer_leave; + Callback.register "lui_ocaml_context_menu_press" context_menu_press; Callback.register "lui_ocaml_text_changed" text_changed; Callback.register "lui_ocaml_submit" submit; Callback.register "lui_ocaml_dismiss" dismiss; diff --git a/app/journal_capture.ml b/app/journal_capture.ml index c0a6644..8f34b91 100644 --- a/app/journal_capture.ml +++ b/app/journal_capture.ml @@ -153,7 +153,22 @@ let can_attach t = ;; let add_attachment t staged = - if can_attach t + let duplicate = + List.exists + (fun owned -> + String.equal + (Journal_asset_import.staged_token owned) + (Journal_asset_import.staged_token staged) + || + match + ( Journal_asset_import.staged_source_identity owned + , Journal_asset_import.staged_source_identity staged ) + with + | Some left, Some right -> String.equal left right + | _ -> false) + t.pending_attachments + in + if can_attach t && not duplicate then ( let t = replace_attempt t in { t with pending_attachments = t.pending_attachments @ [ staged ] }) diff --git a/app/journal_graph_runtime.ml b/app/journal_graph_runtime.ml index e663146..8253009 100644 --- a/app/journal_graph_runtime.ml +++ b/app/journal_graph_runtime.ml @@ -29,6 +29,8 @@ type mutation_kind = | Delete_subtree_mutation type payload = + | Projection_changes of Protocol.v2_change_window list + | Projection_resync | Reference_sources_changed of (string * string option) list | Favorites_loaded of Journal_graph_request.favorites_request @@ -163,6 +165,19 @@ type operation = ; depth : int } | List_favorites of Journal_graph_request.favorites_request + | Favorite_dependency of + { epoch : int64 + ; uuid : Graph.Uuid.t + ; seen : string list + ; page : bool + } + | Favorite_page_candidate of Graph.Uuid.t + | Favorite_memberships of + { epoch : int64 + ; page : Graph.Uuid.t + ; consumed : int + ; seen : string list + } | Graph_info | Admission_info of Journal_graph_request.admission_request | Pull_changes of @@ -208,6 +223,11 @@ type operation = { block_id : string ; page : Projection.page } + | Changed_detail_block of + { block_id : string + ; page : Graph.page + } + | Changed_detail_page of Graph.Uuid.t | Find_block_result | Detail_children of { generation : int64 @@ -296,6 +316,21 @@ type t = ; projected_blocks : (string, Projection.block) Hashtbl.t ; page_tree_interests : (string, page_tree_interest) Hashtbl.t ; children_interests : (string, children_interest) Hashtbl.t + ; detail_pages : (string, Graph.page) Hashtbl.t + ; detail_children : (string, Projection.block_member Graph.page_result) Hashtbl.t + ; mutable favorite_epoch : int64 + ; mutable favorites_initialized : bool + ; mutable favorites_page : Graph.Uuid.t option + ; favorite_dependencies : (string, unit) Hashtbl.t + ; favorite_members : (string, unit) Hashtbl.t + ; favorite_targets : (string, Protocol.v2_favorite_target) Hashtbl.t + ; favorite_blocks : (string, Protocol.v2_block_record) Hashtbl.t + ; favorite_pages : (string, Graph.page) Hashtbl.t + ; favorite_dirty : (string, unit) Hashtbl.t + ; favorite_pending : (string, unit) Hashtbl.t + ; favorite_resolved : (string, unit) Hashtbl.t + ; favorite_loading_changes : (string, unit) Hashtbl.t + ; mutable favorite_changes_overflow : bool ; target_read_epochs : (string, int64) Hashtbl.t ; mutable initial_feed : feed_spec option } @@ -351,6 +386,21 @@ let create ?(localtime = Unix.localtime) () = ; projected_blocks = Hashtbl.create 64 ; page_tree_interests = Hashtbl.create 32 ; children_interests = Hashtbl.create 32 + ; detail_pages = Hashtbl.create 32 + ; detail_children = Hashtbl.create 32 + ; favorite_epoch = 0L + ; favorites_initialized = false + ; favorites_page = None + ; favorite_dependencies = Hashtbl.create 32 + ; favorite_members = Hashtbl.create 32 + ; favorite_targets = Hashtbl.create 32 + ; favorite_blocks = Hashtbl.create 32 + ; favorite_pages = Hashtbl.create 32 + ; favorite_dirty = Hashtbl.create 32 + ; favorite_pending = Hashtbl.create 32 + ; favorite_resolved = Hashtbl.create 32 + ; favorite_loading_changes = Hashtbl.create 32 + ; favorite_changes_overflow = false ; target_read_epochs = Hashtbl.create 32 ; initial_feed = None } @@ -376,6 +426,21 @@ let reset t = Hashtbl.clear t.tag_titles; Hashtbl.clear t.page_tree_interests; Hashtbl.clear t.children_interests; + Hashtbl.clear t.detail_pages; + Hashtbl.clear t.detail_children; + t.favorite_epoch <- Int64.succ t.favorite_epoch; + t.favorites_initialized <- false; + t.favorites_page <- None; + Hashtbl.clear t.favorite_dependencies; + Hashtbl.clear t.favorite_members; + Hashtbl.clear t.favorite_targets; + Hashtbl.clear t.favorite_blocks; + Hashtbl.clear t.favorite_pages; + Hashtbl.clear t.favorite_pending; + Hashtbl.clear t.favorite_dirty; + Hashtbl.clear t.favorite_resolved; + Hashtbl.clear t.favorite_loading_changes; + t.favorite_changes_overflow <- false; Hashtbl.clear t.target_read_epochs; t.initial_feed <- None ;; @@ -1564,6 +1629,183 @@ let schedule_hydration t output = let maximum_reference_targets = 4096 let maximum_reference_sources = 8192 +let register_favorite_dependency t uuid = + let key = Graph.Uuid.to_string uuid in + if + (not (Hashtbl.mem t.favorite_dependencies key)) + && Hashtbl.length t.favorite_dependencies >= maximum_reference_targets + then invalid_arg "Favorite dependencies exceed the bounded read limit."; + Hashtbl.replace t.favorite_dependencies key () +;; + +let favorite_dependency_read ?(refresh = false) t ~page ~seen uuid = + let key = Graph.Uuid.to_string uuid in + register_favorite_dependency t uuid; + if Hashtbl.mem t.favorite_pending key + then ( + if refresh then Hashtbl.replace t.favorite_dirty key (); + None) + else if (not refresh) && Hashtbl.mem t.favorite_resolved key + then None + else ( + if Hashtbl.length t.favorite_pending >= maximum_reference_targets + then invalid_arg "Favorite pending reads exceed the bounded read limit."; + Hashtbl.replace t.favorite_pending key (); + Some + (read + t + (Favorite_dependency { epoch = t.favorite_epoch; uuid; seen; page }) + (if page + then Protocol.V2_get_page { page = uuid; revision = None } + else Protocol.V2_get_block { block = uuid; revision = None }))) +;; + +let recheck_dirty_favorite_dependency t ~uuid ~seen ~page = + let key = Graph.Uuid.to_string uuid in + Hashtbl.remove t.favorite_dirty key; + Hashtbl.remove t.favorite_pending key; + (* The held response predates an owned change. Retire its single slot and + replace it with a current point read before considering its old snapshot. *) + schedule_hydration + t + { requests = + Option.to_list (favorite_dependency_read ~refresh:true t ~page ~seen uuid) + ; responses = [] + } +;; + +let favorite_deleted (block : Graph.block) = + List.filter + (fun (property : Graph.property_summary) -> + String.equal property.ident "logseq.property/deleted-at") + block.properties +;; + +let favorite_block_changed + t + key + (before : Protocol.v2_block_record) + (after : Protocol.v2_block_record) + = + before.block.parent <> after.block.parent + || before.block.page <> after.block.page + || favorite_deleted before.block <> favorite_deleted after.block + || (Hashtbl.mem t.favorite_targets key + && (before.block.title <> after.block.title + || before.task_status <> after.task_status)) +;; + +let favorite_dependency_changed t uuid = + Hashtbl.mem t.favorite_loading_changes (Graph.Uuid.to_string uuid) +;; + +let complete_favorite_dependency_key t key output = + Hashtbl.remove t.favorite_pending key; + Hashtbl.remove t.favorite_dirty key; + Hashtbl.replace t.favorite_resolved key (); + if Hashtbl.length t.favorite_pending = 0 + then ( + Hashtbl.clear t.favorite_loading_changes; + let overflow = t.favorite_changes_overflow in + t.favorite_changes_overflow <- false; + if overflow + then { output with responses = response Favorites_invalidated :: output.responses } + else output) + else output +;; + +let complete_favorite_dependency t uuid output = + complete_favorite_dependency_key t (Graph.Uuid.to_string uuid) output +;; + +let favorite_membership_key uuid = "memberships:" ^ Graph.Uuid.to_string uuid + +let favorite_changes t blocks pages structures = + let membership uuid = Hashtbl.mem t.favorite_members (Graph.Uuid.to_string uuid) in + let favorites_scope uuid = + Option.fold ~none:false ~some:(Graph.Uuid.equal uuid) t.favorites_page + in + let changed = + List.exists membership blocks + || List.exists favorites_scope pages + || List.exists + (function + | Protocol.V2_children_interest parent | V2_page_tree_interest parent -> + favorites_scope parent + | V2_journal_index_interest -> false) + structures + in + let rechecks = + List.filter_map + (fun (uuid, page) -> + if + Hashtbl.mem t.favorite_dependencies (Graph.Uuid.to_string uuid) + && (not (membership uuid)) + && not (favorites_scope uuid) + then favorite_dependency_read ~refresh:true t ~page ~seen:[] uuid + else None) + (List.map (fun uuid -> uuid, false) blocks @ List.map (fun uuid -> uuid, true) pages) + in + if Hashtbl.length t.favorite_pending > 0 + then + List.iter + (fun uuid -> + let key = Graph.Uuid.to_string uuid in + if Hashtbl.length t.favorite_loading_changes < maximum_reference_targets + then Hashtbl.replace t.favorite_loading_changes key () + else t.favorite_changes_overflow <- true) + (blocks @ pages); + let candidates = + if t.favorites_initialized && t.favorites_page = None + then + List.map + (fun page -> + read + t + (Favorite_page_candidate page) + (Protocol.V2_get_page { page; revision = None })) + pages + else [] + in + { requests = rechecks @ candidates + ; responses = (if changed then [ response Favorites_invalidated ] else []) + } +;; + +let remember_detail_members t ~page ~(root : Projection.block_member) children = + let key = Graph.Uuid.to_string root.Projection.block.uuid in + Hashtbl.replace t.detail_pages key page; + List.iter + (fun (member : Projection.block_member) -> + Hashtbl.replace t.detail_pages (Graph.Uuid.to_string member.block.uuid) page) + children.Graph.items; + Hashtbl.replace t.detail_children key children +;; + +let remember_detail_block t (block : Graph.block) revision = + let key = Graph.Uuid.to_string block.uuid in + Option.iter + (fun (interest : children_interest) -> + Hashtbl.replace + t.children_interests + key + { interest with root = { block; revision } }) + (Hashtbl.find_opt t.children_interests key); + Hashtbl.filter_map_inplace + (fun _ children -> + Some + { children with + Graph.items = + List.map + (fun (member : Projection.block_member) -> + if Graph.Uuid.equal member.block.uuid block.uuid + then { Projection.block; revision } + else member) + children.Graph.items + }) + t.detail_children +;; + let reference_read t block_id depth = if Hashtbl.mem t.reference_pending block_id then None @@ -1656,19 +1898,28 @@ let observe_reference_sources t ~source_epoch depth values = let hydration_for_changes t ~request_generation windows = let blocks, pages, structures = changed_interests windows in + let favorites = favorite_changes t blocks pages structures in (* Fence old fragments before issuing the shared or dedicated point reads. *) List.iter (fun block -> fence_reference t (Graph.Uuid.to_string block)) blocks; let point_reads = List.filter_map (fun block -> let block_id = Graph.Uuid.to_string block in - Option.map - (fun page -> - read + match page_by_block t block_id with + | Some page -> + Some + (read t (Changed_block { block_id; page }) (Protocol.V2_get_block { block; revision = None })) - (page_by_block t block_id)) + | None -> + Option.map + (fun page -> + read + t + (Changed_detail_block { block_id; page }) + (Protocol.V2_get_block { block; revision = None })) + (Hashtbl.find_opt t.detail_pages block_id)) blocks in let reference_reads = @@ -1693,6 +1944,21 @@ let hydration_for_changes t ~request_generation windows = | V2_children_interest _ | V2_journal_index_interest -> None) structures in + let detail_page_reads = + List.filter_map + (fun page -> + if + Hashtbl.to_seq_values t.children_interests + |> Seq.exists (fun interest -> Graph.Uuid.equal interest.page.uuid page) + then + Some + (read + t + (Changed_detail_page page) + (Protocol.V2_get_page { page; revision = None })) + else None) + pages + in let children_reads = List.filter_map (function @@ -1739,8 +2005,14 @@ let hydration_for_changes t ~request_generation windows = schedule_hydration t { requests = - point_reads @ reference_reads @ tree_reads @ children_reads @ journal.requests - ; responses = journal.responses + point_reads + @ reference_reads + @ tree_reads + @ children_reads + @ detail_page_reads + @ journal.requests + @ favorites.requests + ; responses = journal.responses @ favorites.responses } ;; @@ -1806,14 +2078,41 @@ let rehydrate_current_interests t ~request_generation ~generation = |> Seq.filter_map (invalidate_reference t) |> List.of_seq in + let detail_roots = + Hashtbl.to_seq_values t.children_interests + |> Seq.map (fun (interest : children_interest) -> + let block_id = Graph.Uuid.to_string interest.root.block.uuid in + read + t + (Changed_detail_block { block_id; page = interest.page }) + (Protocol.V2_get_block { block = interest.root.block.uuid; revision = None })) + |> List.of_seq + in + let detail_pages = Hashtbl.create 16 in + Hashtbl.iter + (fun _ (interest : children_interest) -> + Hashtbl.replace + detail_pages + (Graph.Uuid.to_string interest.page.uuid) + interest.page.uuid) + t.children_interests; + let detail_page_requests = + Hashtbl.to_seq_values detail_pages + |> Seq.map (fun page -> + read t (Changed_detail_page page) (Protocol.V2_get_page { page; revision = None })) + |> List.of_seq + in schedule_hydration t { requests = (graph_info :: journal.requests) @ tree_requests @ children_requests + @ detail_roots + @ detail_page_requests @ reference_requests - ; responses = journal.responses + ; responses = + response Projection_resync :: response Favorites_invalidated :: journal.responses } ;; @@ -1876,6 +2175,9 @@ let read_child_parent t command page child = let operation_name = function | Reference_read _ -> "readBlockReference" | List_favorites _ -> "listFavorites" + | Favorite_dependency _ -> "readFavoriteDependency" + | Favorite_page_candidate _ -> "findFavoritePage" + | Favorite_memberships _ -> "readFavoriteMemberships" | Graph_info -> "graphInfo" | Admission_info _ -> "inspectAdmission" | Pull_changes _ -> "pullChanges" @@ -1890,6 +2192,8 @@ let operation_name = function | Child_status _ -> "setAppendedChildStatus" | Child_parent _ -> "readAppendParent" | Changed_block _ -> "reconcileChangedBlock" + | Changed_detail_block _ -> "reconcileChangedDetailBlock" + | Changed_detail_page _ -> "reconcileChangedDetailPage" | Find_block_result -> "findBlock" | Detail_children _ -> "loadDetailChildren" | Changed_children _ -> "refreshChildren" @@ -1932,6 +2236,22 @@ let failure_output | List_favorites request -> responses [ response (Favorites_failed (request, code = Error.Stale_read_cursor, message)) ] + | Favorite_dependency { epoch; uuid; _ } -> + if epoch <> t.favorite_epoch + then empty + else + complete_favorite_dependency + t + uuid + (responses [ response (Rejected (Worker_failure worker_failure)) ]) + | Favorite_memberships { epoch; page; _ } -> + if epoch <> t.favorite_epoch + then empty + else + complete_favorite_dependency_key + t + (favorite_membership_key page) + (responses [ response (Rejected (Worker_failure worker_failure)) ]) | List_feed_pages { request_generation; _ } -> responses [ response @@ -2007,7 +2327,7 @@ let failure_output Hashtbl.remove t.reference_dirty block_id; requests (Option.to_list (reference_read t block_id depth))) else observe_reference_sources t ~source_epoch depth [ block_id, None ] - | Changed_block { block_id; _ } -> + | Changed_block { block_id; _ } | Changed_detail_block { block_id; _ } -> let invalidation = if Hashtbl.mem t.reference_depths block_id then observe_reference_sources t ~source_epoch 0 [ block_id, None ] @@ -2018,6 +2338,8 @@ let failure_output response (Rejected (Worker_failure worker_failure)) :: invalidation.responses } | Find_block_result + | Favorite_page_candidate _ + | Changed_detail_page _ | Changed_children _ | Capture_page _ | Capture_format _ @@ -2154,7 +2476,78 @@ let receive_response t (protocol_response : Protocol.response) = | Protocol.V2_favorites_outcome result -> (match operation with | List_favorites request -> - responses [ response (Favorites_loaded (request, result)) ] + (try + if request.cursor = None + then ( + t.favorite_epoch <- Int64.succ t.favorite_epoch; + Hashtbl.clear t.favorite_dependencies; + Hashtbl.clear t.favorite_members; + Hashtbl.clear t.favorite_targets; + Hashtbl.clear t.favorite_blocks; + Hashtbl.clear t.favorite_pages; + Hashtbl.clear t.favorite_pending; + Hashtbl.clear t.favorite_dirty; + Hashtbl.clear t.favorite_resolved; + Hashtbl.clear t.favorite_loading_changes; + t.favorite_changes_overflow <- false); + t.favorites_initialized <- true; + t.favorites_page <- result.favorites_page; + Option.iter + (fun uuid -> register_favorite_dependency t uuid) + result.favorites_page; + (* Register the complete page before creating any pending reads. A + limit failure must not leave unscheduled request constructors. *) + List.iter + (fun (item : Protocol.v2_favorite_item) -> + register_favorite_dependency t item.membership_uuid; + match item.target with + | V2_favorite_page { uuid; _ } | V2_favorite_block { uuid; _ } -> + register_favorite_dependency t uuid) + result.items; + let dependencies = + List.filter_map + (fun (item : Protocol.v2_favorite_item) -> + register_favorite_dependency t item.membership_uuid; + Hashtbl.replace + t.favorite_members + (Graph.Uuid.to_string item.membership_uuid) + (); + match item.target with + | V2_favorite_page { uuid; _ } -> + Hashtbl.replace + t.favorite_targets + (Graph.Uuid.to_string uuid) + item.target; + favorite_dependency_read t ~page:true ~seen:[] uuid + | V2_favorite_block { uuid; _ } -> + Hashtbl.replace + t.favorite_targets + (Graph.Uuid.to_string uuid) + item.target; + favorite_dependency_read t ~page:false ~seen:[] uuid) + result.items + in + let memberships = + match request.cursor, result.favorites_page with + | None, Some page -> + Hashtbl.replace t.favorite_pending (favorite_membership_key page) (); + [ read + t + (Favorite_memberships + { epoch = t.favorite_epoch; page; consumed = 0; seen = [] }) + (Protocol.V2_get_children + { parent = page; limit = 200; cursor = None; revision = None }) + ] + | _ -> [] + in + schedule_hydration + t + { requests = dependencies @ memberships + ; responses = [ response (Favorites_loaded (request, result)) ] + } + with + | Invalid_argument message -> + responses [ response (Favorites_failed (request, false, message)) ]) | _ -> failure_output t operation request_id "Unexpected favorites response.") | Protocol.V2_failed { code; message } -> (match Error.code_of_string code with @@ -2267,6 +2660,86 @@ let receive_response t (protocol_response : Protocol.response) = | _ -> failure_output t operation request_id "Unexpected journal response.") | V2_page_outcome lookup -> (match lookup, operation with + | _, Favorite_dependency { epoch; _ } when epoch <> t.favorite_epoch -> empty + | _, Favorite_dependency { uuid; seen; page; _ } + when Hashtbl.mem t.favorite_dirty (Graph.Uuid.to_string uuid) -> + recheck_dirty_favorite_dependency t ~uuid ~seen ~page + | V2_present_page { page; _ }, Favorite_dependency { uuid; page = true; _ } -> + let key = Graph.Uuid.to_string uuid in + let changed = + match Hashtbl.find_opt t.favorite_pages key with + | Some before -> + before.recycled <> page.recycled + || (Hashtbl.mem t.favorite_targets key && before.title <> page.title) + | None -> + page.recycled + || + (match Hashtbl.find_opt t.favorite_targets key with + | Some (V2_favorite_page target) -> target.title <> page.title + | _ -> false) + in + Hashtbl.replace t.favorite_pages key page; + Hashtbl.replace t.graph_pages (Graph.Uuid.to_string page.uuid) page; + complete_favorite_dependency + t + uuid + (if changed then responses [ response Favorites_invalidated ] else empty) + | V2_missing_page _, Favorite_dependency { uuid; page = true; _ } -> + complete_favorite_dependency + t + uuid + (responses [ response Favorites_invalidated ]) + | V2_present_page { page; _ }, Favorite_page_candidate _ -> + if String.equal page.name "$$$favorites" && not page.recycled + then responses [ response Favorites_invalidated ] + else empty + | V2_missing_page _, Favorite_page_candidate _ -> empty + | V2_present_page { page; revision }, Changed_detail_page _ -> + Hashtbl.replace t.graph_pages (Graph.Uuid.to_string page.uuid) page; + remember_page_revision t page.uuid revision; + let reconciled = ref [] in + Hashtbl.filter_map_inplace + (fun key (interest : children_interest) -> + if Graph.Uuid.equal interest.page.uuid page.uuid + then ( + let interest = { interest with page } in + Option.iter + (fun children -> + match projection_time_context t with + | Error _ -> () + | Ok time_context -> + (match + Projection.detail_on_page + ~page + ~time_context + ~root:interest.root + children + with + | Ok detail -> + reconciled + := response (Children_reconciled detail) :: !reconciled + | Error _ -> ())) + (Hashtbl.find_opt t.detail_children key); + Some interest) + else Some interest) + t.children_interests; + Hashtbl.filter_map_inplace + (fun _ (old : Graph.page) -> + Some (if Graph.Uuid.equal old.Graph.uuid page.uuid then page else old)) + t.detail_pages; + responses (List.rev !reconciled) + | V2_missing_page { uuid; _ }, Changed_detail_page _ -> + responses + (Hashtbl.to_seq_values t.children_interests + |> Seq.filter_map (fun (interest : children_interest) -> + if Graph.Uuid.equal interest.page.uuid uuid + then + Some + (response + (Block_removed + { block_id = Graph.Uuid.to_string interest.root.block.uuid })) + else None) + |> List.of_seq) | ( V2_present_page { page; revision } , Detail_page { generation; root; limit; after } ) -> remember_page_revision t page.uuid revision; @@ -2470,6 +2943,63 @@ let receive_response t (protocol_response : Protocol.response) = value.tag_titles | V2_missing_block _ -> ()); (match lookup, operation with + | _, Favorite_dependency { epoch; _ } when epoch <> t.favorite_epoch -> empty + | _, Favorite_dependency { uuid; seen; page; _ } + when Hashtbl.mem t.favorite_dirty (Graph.Uuid.to_string uuid) -> + recheck_dirty_favorite_dependency t ~uuid ~seen ~page + | ( V2_present_block { value; _ } + , Favorite_dependency { uuid; seen; page = false; _ } ) -> + let key = Graph.Uuid.to_string uuid in + let parent = value.block.parent in + if + (not (Graph.Uuid.equal value.block.uuid uuid)) + || List.length seen >= 256 + || List.mem key seen + || List.mem (Graph.Uuid.to_string parent) (key :: seen) + then + complete_favorite_dependency + t + uuid + (reject "Favorite ancestry is cyclic or exceeds the read limit.") + else ( + let changed = + match Hashtbl.find_opt t.favorite_blocks key with + | Some before -> favorite_block_changed t key before value + | None -> + favorite_deleted value.block <> [] + || + (match Hashtbl.find_opt t.favorite_targets key with + | Some (V2_favorite_block target) -> + target.title <> value.block.title + || target.task_status <> value.task_status + | _ -> false) + in + Hashtbl.replace t.favorite_blocks key value; + try + let request = + favorite_dependency_read + t + ~page:(Graph.Uuid.equal parent value.block.page) + ~seen:(key :: seen) + parent + in + let output = + schedule_hydration + t + { requests = Option.to_list request + ; responses = + (if changed then [ response Favorites_invalidated ] else []) + } + in + complete_favorite_dependency t uuid output + with + | Invalid_argument message -> + complete_favorite_dependency t uuid (reject message)) + | V2_missing_block _, Favorite_dependency { uuid; page = false; _ } -> + complete_favorite_dependency + t + uuid + (responses [ response Favorites_invalidated ]) | ( V2_present_block { value; revision } , Detail_block { generation; limit; after; _ } ) -> remember_block_revision t value.block.uuid revision; @@ -2596,6 +3126,7 @@ let receive_response t (protocol_response : Protocol.response) = responses [ response (Block_found None) ] | V2_present_block { value; revision }, Changed_block { block_id; page } -> remember_block_revision t value.block.uuid revision; + remember_detail_block t value.block revision; let child_count = Hashtbl.find_opt t.projected_blocks block_id |> Option.map Journal_model.child_count @@ -2616,13 +3147,44 @@ let receive_response t (protocol_response : Protocol.response) = Hashtbl.replace t.projected_blocks block_id block; responses [ response (Block_updated { block; timeline_entry_update = None }) ])) - | V2_missing_block { uuid; revision }, Changed_block { block_id; _ } -> + | V2_present_block { value; revision }, Changed_detail_block { block_id; page } -> + remember_block_revision t value.block.uuid revision; + remember_detail_block t value.block revision; + let child_count = + Hashtbl.find_opt t.projected_blocks block_id + |> Option.map Journal_model.child_count + |> Option.value ~default:0 + in + (match projection_time_context t with + | Error message -> reject message + | Ok time_context -> + (match + Projection.block_on_page + ~page + ~revision + ~child_count + ~time_context + value.block + with + | Error message -> reject message + | Ok block -> + let block = + Journal_model.with_tag_titles block ~tag_titles:value.tag_titles + in + Hashtbl.replace t.projected_blocks block_id block; + responses + [ response (Block_updated { block; timeline_entry_update = None }) ])) + | V2_missing_block { uuid; revision }, Changed_block { block_id; _ } + | V2_missing_block { uuid; revision }, Changed_detail_block { block_id; _ } -> remember_block_revision t uuid revision; t.block_pages <- List.filter (fun (candidate, _) -> not (String.equal candidate block_id)) t.block_pages; Hashtbl.remove t.projected_blocks block_id; + Hashtbl.remove t.detail_pages block_id; + Hashtbl.remove t.children_interests block_id; + Hashtbl.remove t.detail_children block_id; responses [ response (Block_removed { block_id }) ] | V2_present_block { value; revision }, Capture_block { command; page } -> remember_block_revision t value.block.uuid revision; @@ -2643,6 +3205,7 @@ let receive_response t (protocol_response : Protocol.response) = (match operation with | Detail_children { generation; page; root } -> let children = children_result items next_cursor in + remember_detail_members t ~page ~root children; Option.iter (fun journal_page -> remember_block_page t journal_page (Graph.Uuid.to_string root.block.uuid); @@ -2680,8 +3243,85 @@ let receive_response t (protocol_response : Protocol.response) = ; failure = Projection_failure message }) ])) + | Favorite_memberships { epoch; _ } when epoch <> t.favorite_epoch -> empty + | Favorite_memberships { epoch; page; consumed; seen } -> + if List.length items > 200 || consumed + List.length items > 10_000 + then + complete_favorite_dependency_key + t + (favorite_membership_key page) + (reject "Favorite memberships exceed the bounded read limit.") + else ( + try + List.iter + (fun (item : Protocol.v2_child_member) -> + register_favorite_dependency t item.value.block.uuid; + Hashtbl.replace + t.favorite_members + (Graph.Uuid.to_string item.value.block.uuid) + ()) + items; + let changed = + List.exists + (fun (item : Protocol.v2_child_member) -> + favorite_dependency_changed t item.value.block.uuid) + items + in + let output = + if changed then responses [ response Favorites_invalidated ] else empty + in + match next_cursor with + | None -> + complete_favorite_dependency_key t (favorite_membership_key page) output + | Some cursor -> + let key = Graph.Cursor.to_string cursor in + if + List.mem key seen + || List.length seen >= 128 + || consumed + List.length items >= 10_000 + then + complete_favorite_dependency_key + t + (favorite_membership_key page) + (reject "Favorite membership cursor exceeds the bounded read limit.") + else + schedule_hydration + t + { output with + requests = + [ read + t + (Favorite_memberships + { epoch + ; page + ; consumed = consumed + List.length items + ; seen = key :: seen + }) + (Protocol.V2_get_children + { parent = page + ; limit = 200 + ; cursor = Some cursor + ; revision = None + }) + ] + } + with + | Invalid_argument message -> + complete_favorite_dependency_key + t + (favorite_membership_key page) + (reject message)) | Changed_children { page; root; _ } -> let children = children_result items next_cursor in + let interest = + Hashtbl.find_opt t.children_interests (Graph.Uuid.to_string root.block.uuid) + in + let page, root = + match interest with + | Some latest -> latest.page, latest.root + | None -> page, root + in + remember_detail_members t ~page ~root children; Option.iter (fun journal_page -> remember_block_page t journal_page (Graph.Uuid.to_string root.block.uuid); @@ -2880,7 +3520,7 @@ let receive_response t (protocol_response : Protocol.response) = in let hydration = hydration_for_changes t ~request_generation windows in { requests = acknowledgement :: hydration.requests - ; responses = hydration.responses + ; responses = response (Projection_changes windows) :: hydration.responses } | _ -> failure_output t operation request_id "Unexpected changes response.") | V2_changes_acknowledged _ -> @@ -3017,7 +3657,7 @@ let reconcile_push t ~request_generation push = | V2_resync_required_push { generation; _ } -> rehydrate_current_interests t ~request_generation ~generation in - { output with responses = response Favorites_invalidated :: output.responses } + output ;; module Copy = struct diff --git a/app/journal_graph_runtime.mli b/app/journal_graph_runtime.mli index 262d474..d786443 100644 --- a/app/journal_graph_runtime.mli +++ b/app/journal_graph_runtime.mli @@ -24,6 +24,8 @@ type mutation_kind = | Delete_subtree_mutation type payload = + | Projection_changes of Logseq_db_worker.Protocol.v2_change_window list + | Projection_resync | Reference_sources_changed of (string * string option) list | Favorites_loaded of Journal_graph_request.favorites_request diff --git a/app/journal_lui_bridge.c b/app/journal_lui_bridge.c index 832d45e..f5ed4e4 100644 --- a/app/journal_lui_bridge.c +++ b/app/journal_lui_bridge.c @@ -137,6 +137,51 @@ LUI_EXPORT int32_t lui_ocaml_long_press(int64_t node) { return dispatch_long("lui_ocaml_long_press", node); } +/* Root every allocated pointer argument before subsequent allocations. The + patch callback executes only after dropping these roots and the runtime. */ +static int dispatch_pointer(const char *name, int64_t node, double x, double y, + int32_t modifiers, int32_t button, + const char *target_class) { + patch_response response = {0, NULL, NULL}; + caml_leave_blocking_section(); + const value *dispatch = caml_named_value(name); + if (dispatch != NULL && target_class != NULL) { + CAMLparam0(); + CAMLlocalN(arguments, 6); + CAMLlocal1(result); + arguments[0] = Val_long(node); + arguments[1] = caml_copy_double(x); + arguments[2] = caml_copy_double(y); + arguments[3] = Val_long(modifiers); + arguments[4] = Val_long(button); + arguments[5] = caml_copy_string(target_class); + result = caml_callbackN_exn(*dispatch, 6, arguments); + response = copy_patch(name, result); + CAMLdrop; + } + return release_and_deliver(response); +} + +#define JOURNAL_POINTER_ENTRY(entry) \ + LUI_EXPORT int32_t entry(int64_t node, double x, double y, \ + int32_t modifiers, int32_t button, \ + const char *target_class) { \ + return dispatch_pointer(#entry, node, x, y, modifiers, button, target_class); \ + } + +JOURNAL_POINTER_ENTRY(lui_ocaml_press_detail) +JOURNAL_POINTER_ENTRY(lui_ocaml_pointer_down) +JOURNAL_POINTER_ENTRY(lui_ocaml_pointer_up) +JOURNAL_POINTER_ENTRY(lui_ocaml_context_menu_press) + +LUI_EXPORT int32_t lui_ocaml_pointer_enter(int64_t node) { + return dispatch_long("lui_ocaml_pointer_enter", node); +} + +LUI_EXPORT int32_t lui_ocaml_pointer_leave(int64_t node) { + return dispatch_long("lui_ocaml_pointer_leave", node); +} + static int dispatch_string(const char *name, int64_t node, const char *text) { patch_response response = {0, NULL, NULL}; caml_leave_blocking_section(); diff --git a/app/journal_media.ml b/app/journal_media.ml index 12139d7..b3cb433 100644 --- a/app/journal_media.ml +++ b/app/journal_media.ml @@ -10,6 +10,7 @@ type ticket = type presentation = | Hidden | Placeholder of string + | Failed of string | File of string | External of string @@ -107,10 +108,6 @@ let release t = | _ -> [] ;; -let same_version (a : Asset.t) (b : Asset.t) = - a.uuid = b.uuid && a.source = b.source && a.current_checksum = b.current_checksum -;; - let demand selected = Demand { graph_generation = selected.graph_generation @@ -217,7 +214,7 @@ let status = function | Waiting_unlock -> Placeholder "Unlock graph to view file" | Ready _ -> Placeholder "Opening file" | Failed { failure; _ } -> - Placeholder + Failed (match failure with | Network -> "Unable to download file" | Not_found -> "File is not available on the server" @@ -229,12 +226,20 @@ let status = function ;; let step t = function - | Demand_backpressured -> { t with pressured = true }, [] + | Demand_backpressured -> + (match t.status with + | File _ | Failed _ -> t, [] + | _ -> { t with pressured = true }, []) | Demand_accepted -> { t with pressured = false }, [] | Capacity_available -> (match t.selection with - | Some selected when t.pressured && managed selected.asset -> - { t with pressured = false }, [ demand selected ] + | Some selected + when t.pressured + && managed selected.asset + && + match t.status with + | File _ | Failed _ -> false + | _ -> true -> { t with pressured = false }, [ demand selected ] | _ -> t, []) | Show { graph_generation; consumer; asset } -> let selected = { graph_generation; consumer; asset } in @@ -242,7 +247,7 @@ let step t = function | Some old when old.graph_generation = graph_generation && old.consumer = consumer - && same_version old.asset asset -> { t with selection = Some selected }, [] + && old.asset.uuid = asset.uuid -> { t with selection = Some selected }, [] | _ -> let effects = release_file t @ release t @ if managed asset then [ demand selected ] else [] @@ -263,8 +268,9 @@ let step t = function when selected.graph_generation = scope.graph_generation && selected.consumer = consumer && managed selected.asset -> - (match availability with - | Service.Asset.Ready handle -> + (match t.status, availability with + | File _, _ -> t, [] + | _, Service.Asset.Ready handle -> if (match t.lease with | Some l -> l.ticket.scope = scope && l.ticket.handle = handle @@ -284,11 +290,12 @@ let step t = function ; status = Placeholder "Opening file" } , release_file t @ [ Acquire ticket ] )) - | _ -> + | _, _ -> ( { t with pending = None ; available = None ; lease = None + ; pressured = false ; status = status availability } , release_file t )) @@ -299,7 +306,7 @@ let step t = function match result with | Some (lease, path) -> { t with pending = None; lease = Some { ticket; lease }; status = File path }, [] - | None -> { t with pending = None; status = Placeholder "Unable to open file" }, []) + | None -> { t with pending = None; status = Failed "Unable to open file" }, []) else ( t , (match result, t.lease with @@ -309,7 +316,12 @@ let step t = function | None, _ -> []) ) | Retry_requested -> (match t.selection with - | Some selected when managed selected.asset -> + | Some selected + when managed selected.asset + && + match t.status with + | Failed _ -> true + | _ -> false -> (match t.available with | Some (scope, handle) -> let ticket = { id = t.serial + 1; scope; handle } in diff --git a/app/journal_media.mli b/app/journal_media.mli index 63729fb..612f2b2 100644 --- a/app/journal_media.mli +++ b/app/journal_media.mli @@ -10,6 +10,7 @@ type ticket = private type presentation = | Hidden | Placeholder of string + | Failed of string | File of string | External of string diff --git a/app/journal_media_runtime.ml b/app/journal_media_runtime.ml index 4064d1b..4440af1 100644 --- a/app/journal_media_runtime.ml +++ b/app/journal_media_runtime.ml @@ -10,7 +10,11 @@ type ticket = { root : string ; epoch : int ; request_id : G.Uuid.t - ; continuation : bool + } + | Facts of + { block : string + ; request_id : G.Uuid.t + ; generation : int option } | Lease of { root : string @@ -33,7 +37,7 @@ type view = } type controller = - { asset : Asset.t + { mutable asset : Asset.t ; consumer : string ; mutable state : P.t ; mutable shown : bool @@ -45,7 +49,9 @@ type group = ; epoch : int ; mutable visible : bool ; mutable owners : Owners.t - ; mutable dirty : bool + ; mutable membership_changed : bool + ; mutable invalidation : int + ; mutable dependency_failed : bool ; mutable pending : G.Uuid.t option ; mutable cursor : G.Cursor.t option ; mutable error : string option @@ -60,12 +66,30 @@ type outgoing = ; current : unit -> bool } +type fact = + { value : G.block option + ; issued : int + ; edges_changed : bool + } + +type page_fact = + { alive : bool + ; issued : int + } + type t = { send : ticket option -> Service.request -> bool ; changed : string -> view -> unit ; groups : (string, group) Hashtbl.t ; consumers : (string, group * controller) Hashtbl.t ; previews : (string, string * string * string) Hashtbl.t + ; facts : (string, fact) Hashtbl.t + ; pages : (string, page_fact) Hashtbl.t + ; page_barriers : (string, int) Hashtbl.t + ; sources : (G.Uuid.t, int) Hashtbl.t + ; barriers : (string, int) Hashtbl.t + ; fact_requests : (string, G.Uuid.t) Hashtbl.t + ; mutable clock : int ; mutable generation : int option ; mutable serial : int ; mutable queued : outgoing list @@ -77,6 +101,13 @@ let create ~send ~changed = ; groups = Hashtbl.create 16 ; consumers = Hashtbl.create 32 ; previews = Hashtbl.create 4 + ; facts = Hashtbl.create 64 + ; pages = Hashtbl.create 16 + ; page_barriers = Hashtbl.create 16 + ; sources = Hashtbl.create 64 + ; barriers = Hashtbl.create 64 + ; fact_requests = Hashtbl.create 64 + ; clock = 0 ; generation = None ; serial = 0 ; queued = [] @@ -127,16 +158,95 @@ let notify t g = t.changed g.root view) ;; +let tick t = + t.clock <- t.clock + 1; + t.clock +;; + +let forget_request t request_id = Hashtbl.remove t.sources request_id + +let group_page t g = + match Hashtbl.find_opt t.facts g.root with + | Some { value = Some block; _ } -> Some (G.Uuid.to_string block.page) + | _ -> None +;; + +let group_reaches t g target = + let rec reachable depth seen node = + if node = target + then true + else if depth >= 256 || Owners.mem node seen + then false + else ( + match Hashtbl.find_opt t.facts node with + | Some { value = Some block; _ } -> + let parent = G.Uuid.to_string block.parent in + parent <> G.Uuid.to_string block.page + && reachable (depth + 1) (Owners.add node seen) parent + | _ -> false) + in + if String.starts_with ~prefix:"page:" target + then group_page t g = Some (String.sub target 5 (String.length target - 5)) + else reachable 0 Owners.empty g.root +;; + +let dependency_failure t node message = + Hashtbl.iter + (fun _ g -> + if group_reaches t g node + then ( + g.dependency_failed <- true; + g.error <- Some message; + notify t g)) + t.groups +;; + +let observe_request t (request : Protocol.request) = + if Hashtbl.mem t.sources request.request_id || Hashtbl.length t.sources < 4096 + then Hashtbl.replace t.sources request.request_id (tick t) + else ( + let fail uuid = + dependency_failure + t + (G.Uuid.to_string uuid) + "Attachment dependency requests are full. Retry." + in + match request.command with + | Protocol.V2_get_block { block; _ } | V2_get_block_summary { block; _ } -> fail block + | V2_get_children { parent; _ } -> fail parent + | V2_get_page { page; _ } -> + dependency_failure + t + ("page:" ^ G.Uuid.to_string page) + "Attachment page requests are full. Retry." + | V2_list_assets { roots; _ } -> List.iter fail roots + | _ -> ()) +;; + +let fact_needed t target = + Hashtbl.to_seq_values t.groups + |> Seq.exists (fun g -> + g.visible && (not g.dependency_failed) && group_reaches t g target) +;; + let rec pump t = match t.queued with | [] -> () | next :: rest -> if not (next.current ()) then ( + (match next.ticket with + | Some (Facts { block; request_id; _ }) + when Hashtbl.find_opt t.fact_requests block = Some request_id -> + Hashtbl.remove t.fact_requests block + | _ -> ()); t.queued <- rest; pump t) else if t.send next.ticket next.request then ( + (match next.request with + | Service.Graph_request request -> observe_request t request + | _ -> ()); t.queued <- rest; pump t) ;; @@ -224,22 +334,16 @@ let next_request_id t = ;; let read t g cursor = - if g.pending = None && List.length t.queued < 1024 + if (not g.dependency_failed) && g.pending = None && List.length t.queued < 1024 then ( let request_id = next_request_id t in let root = G.Uuid.of_string g.root |> Result.get_ok in g.pending <- Some request_id; - g.dirty <- false; + g.membership_changed <- false; g.error <- None; enqueue t - ~ticket: - (Query - { root = g.root - ; epoch = g.epoch - ; request_id - ; continuation = Option.is_some cursor - }) + ~ticket:(Query { root = g.root; epoch = g.epoch; request_id }) ~current:(fun () -> current_group t g && g.pending = Some request_id) (Service.Graph_request { api_version = 2 @@ -251,7 +355,7 @@ let read t g cursor = then g.error <- Some "Attachment requests are busy. Retry shortly." ;; -let ensure_fresh t g = if g.dirty && g.pending = None then read t g None +let ensure_fresh t g = if g.membership_changed && g.pending = None then read t g None let reset t ~graph_generation = Hashtbl.iter @@ -262,6 +366,12 @@ let reset t ~graph_generation = Hashtbl.clear t.groups; Hashtbl.clear t.consumers; Hashtbl.clear t.previews; + Hashtbl.clear t.facts; + Hashtbl.clear t.pages; + Hashtbl.clear t.page_barriers; + Hashtbl.clear t.sources; + Hashtbl.clear t.barriers; + Hashtbl.clear t.fact_requests; t.generation <- graph_generation; pump t ;; @@ -279,6 +389,20 @@ let root_visible ?(owner = "default") t ~root visible = | Some g, false, _ -> g.owners <- Owners.remove owner g.owners; g.visible <- not (Owners.is_empty g.owners); + if not g.visible + then ( + let removed = + List.exists + (fun queued -> + match queued.ticket with + | Some (Query { request_id; _ }) -> g.pending = Some request_id + | _ -> false) + t.queued + in + if removed + then ( + g.pending <- None; + g.membership_changed <- true)); List.iter (fun (c : controller) -> c.owners <- Owners.remove owner c.owners; @@ -304,7 +428,9 @@ let root_visible ?(owner = "default") t ~root visible = ; epoch = t.serial ; visible = true ; owners = Owners.singleton owner - ; dirty = true + ; membership_changed = true + ; invalidation = 0 + ; dependency_failed = false ; pending = None ; cursor = None ; error = None @@ -415,13 +541,406 @@ let preview_visible t ~owner ~slot ~root ~asset visible = let next t ~root = match Hashtbl.find_opt t.groups root with - | Some g when g.pending = None && (g.dirty || g.cursor <> None) -> - read t g (if g.dirty then None else g.cursor); + | Some g when g.pending = None && (g.membership_changed || g.cursor <> None) -> + read t g (if g.membership_changed then None else g.cursor); notify t g; pump t | _ -> () ;; +let property_dependencies properties = + let rec internal depth found = function + | G.Internal_uuid uuid -> Owners.add (G.Uuid.to_string uuid) found + | Internal_list values when depth < 64 -> + List.fold_left (internal (depth + 1)) found values + | Internal_map values when depth < 64 -> + List.fold_left + (fun acc (k, v) -> internal (depth + 1) (internal (depth + 1) acc k) v) + found + values + | _ -> found + in + List.fold_left + (fun found (property : G.property_summary) -> + List.fold_left + (fun found -> function + | G.Node_value uuid + | Asset_value uuid + | Entity_value uuid + | Class_value uuid + | Page_value uuid -> Owners.add (G.Uuid.to_string uuid) found + | Map_value values -> internal 0 found (G.Internal_map values) + | Collection_value values -> internal 0 found (G.Internal_list values) + | Any_value value -> internal 0 found value + | _ -> found) + (Owners.add (G.Uuid.to_string property.uuid) found) + property.values) + Owners.empty + properties +;; + +let request_fact t block = + if + (not (Hashtbl.mem t.fact_requests block)) + && Hashtbl.length t.fact_requests < 64 + && List.length t.queued < 1024 + then ( + let page = String.starts_with ~prefix:"page:" block in + let uuid_string = + if page then String.sub block 5 (String.length block - 5) else block + in + match G.Uuid.of_string uuid_string with + | Error _ -> () + | Ok uuid -> + let request_id = next_request_id t in + Hashtbl.add t.fact_requests block request_id; + let generation = t.generation in + enqueue + t + ~ticket:(Facts { block; request_id; generation }) + ~current:(fun () -> + t.generation = generation + && fact_needed t block + && Hashtbl.find_opt t.fact_requests block = Some request_id) + (Service.Graph_request + { api_version = 2 + ; request_id + ; command = + (if page + then V2_get_page { page = uuid; revision = None } + else V2_get_block { block = uuid; revision = None }) + })) +;; + +let request_pending_pages t = + Hashtbl.iter + (fun page at -> + let key = "page:" ^ page in + if + fact_needed t key + && + match Hashtbl.find_opt t.pages page with + | Some fact -> fact.issued <= at + | None -> true + then request_fact t key) + t.page_barriers +;; + +let asset_dependencies t g = + let successful = + List.fold_left + (fun ids c -> + match P.presentation c.state with + | P.File _ -> Owners.add (G.Uuid.to_string c.asset.uuid) ids + | _ -> ids) + Owners.empty + g.controllers + in + let found = + match Hashtbl.find_opt t.facts g.root with + | Some { value = Some block; _ } -> + List.fold_left + (fun deps uuid -> Owners.add (G.Uuid.to_string uuid) deps) + (property_dependencies block.properties) + block.refs + | _ -> Owners.empty + in + let found = + List.fold_left + (fun deps c -> Owners.add (G.Uuid.to_string c.asset.uuid) deps) + found + g.controllers + in + Owners.diff found successful +;; + +let membership_signature block = + block.G.parent, block.refs, block.tags, property_dependencies block.properties +;; + +let truncated (block : G.block) = + List.exists (fun (p : G.property_summary) -> p.values_truncated) block.G.properties +;; + +let dependencies t g = + let rec ancestors depth seen node = + if depth >= 256 || Owners.mem node seen + then seen + else ( + let seen = Owners.add node seen in + match Hashtbl.find_opt t.facts node with + | None -> + if g.visible && not g.dependency_failed then request_fact t node; + seen + | Some { value = None; _ } -> seen + | Some { value = Some block; _ } -> + let page = G.Uuid.to_string block.page + and parent = G.Uuid.to_string block.parent in + let seen = Owners.add page seen in + if parent = page then seen else ancestors (depth + 1) seen parent) + in + let found = ancestors 0 Owners.empty g.root in + Owners.union found (asset_dependencies t g) +;; + +let invalidate t g at = + if at > g.invalidation + then ( + g.invalidation <- at; + g.membership_changed <- true; + g.cursor <- None; + if g.visible && not g.dependency_failed then ensure_fresh t g) +;; + +let reconcile_dependencies t = + Hashtbl.iter + (fun _ g -> + let deps = dependencies t g + and assets = asset_dependencies t g in + let latest = + Owners.fold + (fun node at -> + let changed = + Owners.mem node assets + || + match Hashtbl.find_opt t.facts node with + | Some fact -> fact.edges_changed + | None -> false + in + if changed + then max at (Option.value (Hashtbl.find_opt t.barriers node) ~default:0) + else at) + deps + 0 + in + invalidate t g latest) + t.groups; + request_pending_pages t +;; + +let observe_response t (Protocol.V2_response { request_id; outcome; _ }) = + match Hashtbl.find_opt t.sources request_id with + | None -> () + | Some issued -> + Hashtbl.remove t.sources request_id; + let changed = ref false in + let remember uuid value = + let key = G.Uuid.to_string uuid in + let barrier = Option.value (Hashtbl.find_opt t.barriers key) ~default:0 in + let prior = + match Hashtbl.find_opt t.facts key with + | Some fact -> fact.issued + | None -> 0 + in + if + issued > barrier + && issued >= prior + && (Hashtbl.mem t.facts key || Hashtbl.length t.facts < 4096) + then ( + let edges_changed = + match Hashtbl.find_opt t.facts key, value with + | Some { value = Some old; _ }, Some fresh -> + membership_signature old <> membership_signature fresh + || truncated old + || truncated fresh + | Some { value = None; _ }, None -> false + | Some _, _ -> true + | None, _ -> + Hashtbl.to_seq_values t.groups |> Seq.exists (fun g -> group_reaches t g key) + in + Hashtbl.replace t.facts key { value; issued; edges_changed }; + if Hashtbl.mem t.barriers key then changed := true) + else if issued > barrier && issued >= prior && not (Hashtbl.mem t.facts key) + then dependency_failure t key "Attachment dependency cache is full. Retry." + in + let remember_page uuid alive = + let page = G.Uuid.to_string uuid in + let barrier = Option.value (Hashtbl.find_opt t.page_barriers page) ~default:0 in + let previous = Hashtbl.find_opt t.pages page in + let prior = + match previous with + | Some fact -> fact.issued + | None -> 0 + in + if issued > barrier && issued >= prior + then + if Hashtbl.mem t.pages page || Hashtbl.length t.pages < 64 + then ( + let was_alive = + match previous with + | Some fact -> fact.alive + | None -> true + in + Hashtbl.replace t.pages page { alive; issued }; + if was_alive <> alive + then + Hashtbl.iter + (fun _ g -> + if group_page t g = Some page then invalidate t g (max barrier issued)) + t.groups) + else dependency_failure t ("page:" ^ page) "Attachment page cache is full. Retry." + in + let page_lookup = function + | Protocol.V2_present_page { page; _ } -> + remember_page page.uuid (not page.recycled) + | V2_missing_page { uuid; _ } -> remember_page uuid false + in + let block (record : Protocol.v2_block_record) = + remember record.block.uuid (Some record.block) + in + let lookup = function + | Protocol.V2_present_block { value; _ } -> block value + | V2_missing_block { uuid; _ } -> remember uuid None + in + (match outcome with + | Protocol.V2_block_outcome value -> lookup value + | V2_block_summary_outcome { lookup = value; page; items; _ } -> + lookup value; + Option.iter page_lookup page; + List.iter (fun (item : Protocol.v2_child_member) -> block item.value) items + | V2_children_outcome { items; _ } -> + List.iter (fun (item : Protocol.v2_child_member) -> block item.value) items + | V2_page_tree_outcome { items; _ } -> + List.iter (fun (item : Protocol.v2_tree_member) -> block item.value) items + | V2_page_outcome value -> page_lookup value + | V2_journals_outcome { items; _ } -> + List.iter + (fun (item : Protocol.v2_journal_item) -> + remember_page item.page.uuid (not item.page.recycled)) + items + | _ -> ()); + if !changed + then ( + reconcile_dependencies t; + pump t) +;; + +(* Observation registers facts only. The changes owner determines which + membership reads are invalid; dependency lookups complete that decision. *) + +let chain_complete t g = + let rec walk depth seen node = + if depth >= 256 || Owners.mem node seen + then true + else ( + match Hashtbl.find_opt t.facts node with + | None -> false + | Some { value = None; _ } -> true + | Some { value = Some block; _ } -> + let parent = G.Uuid.to_string block.parent in + parent = G.Uuid.to_string block.page + || walk (depth + 1) (Owners.add node seen) parent) + in + walk 0 Owners.empty g.root +;; + +let changes t windows = + let at = tick t in + let relevant = + Hashtbl.fold + (fun _ g deps -> Owners.union deps (dependencies t g)) + t.groups + Owners.empty + in + let unresolved = + Hashtbl.to_seq_values t.groups + |> Seq.exists (fun g -> (not g.dependency_failed) && not (chain_complete t g)) + in + if not unresolved + then + Hashtbl.filter_map_inplace + (fun node barrier -> if Owners.mem node relevant then Some barrier else None) + t.barriers; + let mark uuid = + let key = G.Uuid.to_string uuid in + if Owners.mem key relevant || unresolved + then + if Hashtbl.mem t.barriers key || Hashtbl.length t.barriers < 4096 + then Hashtbl.replace t.barriers key at + else + Hashtbl.iter + (fun _ g -> + if group_reaches t g key || not (chain_complete t g) + then ( + g.dependency_failed <- true; + g.error <- Some "Attachment change dependencies are full. Retry."; + notify t g)) + t.groups + in + (* Non-recursive membership does not depend on siblings or page-tree growth. + The changed block identities carry holder/ancestor moves and deletion. *) + List.iter + (fun (window : Protocol.v2_change_window) -> List.iter mark window.block_uuids) + windows; + List.iter + (fun (window : Protocol.v2_change_window) -> + List.iter + (fun uuid -> + let page = G.Uuid.to_string uuid in + if + Hashtbl.to_seq_values t.groups + |> Seq.exists (fun g -> group_page t g = Some page) + then + if Hashtbl.mem t.page_barriers page || Hashtbl.length t.page_barriers < 64 + then Hashtbl.replace t.page_barriers page at + else + dependency_failure + t + ("page:" ^ page) + "Attachment page dependencies are full. Retry.") + window.page_uuids) + windows; + reconcile_dependencies t; + Hashtbl.iter + (fun _ g -> + if g.visible && not g.dependency_failed + then + Owners.iter + (fun node -> + if + Hashtbl.find_opt t.barriers node = Some at + && Hashtbl.mem t.facts node + && not (Owners.mem node (asset_dependencies t g)) + then ( + let fact = Hashtbl.find t.facts node in + let stable_file = + node = g.root + && List.exists + (fun c -> + G.Uuid.to_string c.asset.uuid = node + && + match P.presentation c.state with + | P.File _ -> true + | _ -> false) + g.controllers + in + if (not fact.edges_changed) && not stable_file then request_fact t node)) + (dependencies t g)) + t.groups; + Hashtbl.iter + (fun node fact -> + if Hashtbl.find_opt t.barriers node = Some at + then Hashtbl.replace t.facts node { fact with edges_changed = false }) + t.facts; + pump t +;; + +let resync t = + let at = tick t in + Hashtbl.clear t.facts; + Hashtbl.clear t.pages; + Hashtbl.clear t.page_barriers; + Hashtbl.clear t.sources; + Hashtbl.clear t.fact_requests; + Hashtbl.clear t.barriers; + Hashtbl.iter + (fun _ g -> + g.dependency_failed <- false; + invalidate t g at) + t.groups; + pump t +;; + let retry ?(owner = "default") t ~root ~asset = match Hashtbl.find_opt t.groups root with | None -> () @@ -429,22 +948,36 @@ let retry ?(owner = "default") t ~root ~asset = (match List.find_opt (fun c -> c.consumer = asset) g.controllers with | Some c when c.shown -> dispatch t g c Retry_requested | Some _ -> asset_visible ~owner t ~root ~asset true - | None when asset = "" -> read t g None + | None when asset = "" -> + (* Explicit retry may reclaim facts no retained root depends on. *) + Hashtbl.filter_map_inplace + (fun node fact -> + if + Hashtbl.to_seq_values t.groups + |> Seq.exists (fun owner -> group_reaches t owner node) + then Some fact + else None) + t.facts; + let retained_page page = + Hashtbl.to_seq_values t.groups + |> Seq.exists (fun owner -> group_page t owner = Some page) + in + Hashtbl.filter_map_inplace + (fun page fact -> if retained_page page then Some fact else None) + t.pages; + Hashtbl.filter_map_inplace + (fun page barrier -> if retained_page page then Some barrier else None) + t.page_barriers; + g.dependency_failed <- false; + g.error <- None; + ignore (dependencies t g); + request_pending_pages t; + read t g None | None -> ()); notify t g; pump t ;; -let refresh t = - Hashtbl.iter - (fun _ g -> - g.dirty <- true; - g.cursor <- None; - if g.visible then ensure_fresh t g) - t.groups; - pump t -;; - let release_stale t ticket result = match result with | None -> () @@ -454,12 +987,50 @@ let release_stale t ticket result = let receive t ticket response = (match ticket with - | Query { root; epoch; request_id; continuation } -> + | Facts { block; request_id; generation } -> + if + t.generation = generation + && Hashtbl.find_opt t.fact_requests block = Some request_id + then ( + Hashtbl.remove t.fact_requests block; + match response with + | Service.Graph_response + (Protocol.V2_response { request_id = actual; outcome; _ } as raw) + when actual = request_id -> + let barrier = + if String.starts_with ~prefix:"page:" block + then + Hashtbl.find_opt + t.page_barriers + (String.sub block 5 (String.length block - 5)) + else Hashtbl.find_opt t.barriers block + in + let superseded = + match Hashtbl.find_opt t.sources request_id, barrier with + | Some issued, Some at -> issued <= at + | _ -> false + in + observe_response t raw; + (match outcome with + | V2_block_outcome _ when superseded || Hashtbl.mem t.facts block -> () + | V2_page_outcome _ when String.starts_with ~prefix:"page:" block -> () + | _ -> + dependency_failure t block "Unable to load attachment dependencies. Retry."); + reconcile_dependencies t; + (match outcome with + | V2_block_outcome _ when superseded && fact_needed t block -> + request_fact t block + | _ -> ()) + | _ -> + Hashtbl.remove t.sources request_id; + dependency_failure t block "Unable to load attachment dependencies. Retry.") + | Query { root; epoch; request_id } -> (match Hashtbl.find_opt t.groups root with | Some g when g.epoch = epoch && g.pending = Some request_id -> + Hashtbl.remove t.sources request_id; g.pending <- None; - if g.dirty - then (if g.visible then ensure_fresh t g) + if g.membership_changed + then (if g.visible && not g.dependency_failed then ensure_fresh t g) else ( match response with | Service.Graph_response @@ -473,8 +1044,20 @@ let receive t ticket response = let controllers = List.map (fun asset -> - match List.find_opt (fun c -> c.asset = asset) old with - | Some c -> c + match List.find_opt (fun c -> c.asset.uuid = asset.Asset.uuid) old with + | Some c -> + c.asset <- asset; + if c.shown + then ( + match t.generation with + | Some graph_generation -> + dispatch + t + g + c + (P.Show { graph_generation; consumer = c.consumer; asset }) + | None -> ()); + c | None -> t.serial <- t.serial + 1; { asset @@ -496,19 +1079,8 @@ let receive t ticket response = old; g.controllers <- controllers; g.cursor <- next_cursor; - g.error <- None; + if not g.dependency_failed then g.error <- None; List.iter (fun c -> Hashtbl.replace t.consumers c.consumer (g, c)) controllers - | Service.Graph_response - (Protocol.V2_response - { request_id = actual; outcome = V2_failed { code; _ }; _ }) - when actual = request_id - && code = Logseq_db_worker.Error.code_string Stale_read_cursor -> - g.cursor <- None; - if continuation - then ( - g.dirty <- true; - if g.visible then ensure_fresh t g) - else g.error <- Some "Unable to load attachments. Retry." | _ -> g.error <- Some "Unable to load attachments. Retry."); notify t g | _ -> ()) @@ -568,7 +1140,9 @@ let imported t ~current (receipt : Logseq_db_worker.import_receipt) = ; epoch = t.serial ; visible = false ; owners = Owners.empty - ; dirty = true + ; membership_changed = true + ; invalidation = 0 + ; dependency_failed = false ; pending = None ; cursor = None ; error = None diff --git a/app/journal_media_runtime.mli b/app/journal_media_runtime.mli index 8b38e22..899ecb2 100644 --- a/app/journal_media_runtime.mli +++ b/app/journal_media_runtime.mli @@ -1,5 +1,6 @@ module Service = Logseq_db_worker_lui.Logseq_db_worker_lui_service module Asset = Logseq_db_types.Asset_descriptor +module Protocol = Logseq_db_worker.Protocol type ticket @@ -46,13 +47,15 @@ val preview_visible val next : t -> root:string -> unit val retry : ?owner:string -> t -> root:string -> asset:string -> unit +val observe_response : t -> Protocol.response -> unit -(** Mark retained metadata dirty after a projection change or resync. Only - active root/asset/preview owners read immediately; hidden groups restart - lazily. Each root keeps one request and at most one dirty follow-up, always - from the first page because cursors belong to the global projection. *) -val refresh : t -> unit +(** Register a successfully admitted current-scope query before its response. + Observations retire their registration and cannot overwrite newer facts. *) +val observe_request : t -> Protocol.request -> unit +val forget_request : t -> Logseq_db_types.Graph_types.Uuid.t -> unit +val changes : t -> Protocol.v2_change_window list -> unit +val resync : t -> unit val receive : t -> ticket -> Service.response -> unit val reject : t -> ticket -> unit val notice : t -> Service.asset_scope -> Service.asset_notice -> unit diff --git a/app/journal_media_view.ml b/app/journal_media_view.ml index 64e50f4..fa9260a 100644 --- a/app/journal_media_view.ml +++ b/app/journal_media_view.ml @@ -250,7 +250,7 @@ let view_content match current.presentation with | Journal_media.File path -> Some (owner, candidate.token, path, root ^ ":" ^ candidate.token) - | Hidden | Placeholder _ | External _ -> None))) + | Hidden | Placeholder _ | Failed _ | External _ -> None))) image_slots else [ selected ] in @@ -379,7 +379,7 @@ let view_content ~accessibility_identifier:id ~on_appear:visible [] - | Placeholder message -> + | Placeholder message | Failed message -> L.column ~key:item.token ~gap:6 @@ -388,14 +388,18 @@ let view_content ~background:"#839B7F0B" ~corner_radius:10 ~on_appear:visible - [ L.text ~value:message ~style_class:"caption" ~foreground:"secondary" [] - ; L.text - ~value:"Retry" - ~style_class:"caption" - ~on_press:(fun _ -> - emit ~target_root:(asset_root item.token) ~asset:item.token Retry) - [] - ] + ([ L.text ~value:message ~style_class:"caption" ~foreground:"secondary" [] ] + @ + match item.presentation with + | Failed _ -> + [ L.text + ~value:"Retry" + ~style_class:"caption" + ~on_press:(fun _ -> + emit ~target_root:(asset_root item.token) ~asset:item.token Retry) + [] + ] + | _ -> []) | Hidden -> L.text ~key:item.token @@ -413,7 +417,7 @@ let view_content | Some item -> let message = match item.presentation with - | Placeholder message -> message + | Placeholder message | Failed message -> message | Hidden -> "Waiting for file" | External _ | File _ -> "Waiting for file" in @@ -423,18 +427,23 @@ let view_content ~cross:`start ~on_appear:(fun _ -> emit ~target_root:(asset_root item.token) ~asset:item.token Asset) - [ L.text - ~value:message - ~style_class:"caption line-clamp-3" - ~foreground:"secondary" - [] - ; L.text - ~value:"Retry" - ~style_class:"caption" - ~on_press:(fun _ -> - emit ~target_root:(asset_root item.token) ~asset:item.token Retry) - [] - ] + ([ L.text + ~value:message + ~style_class:"caption line-clamp-3" + ~foreground:"secondary" + [] + ] + @ + match item.presentation with + | Failed _ -> + [ L.text + ~value:"Retry" + ~style_class:"caption" + ~on_press:(fun _ -> + emit ~target_root:(asset_root item.token) ~asset:item.token Retry) + [] + ] + | _ -> []) | None -> L.column ~padding:10 diff --git a/app/journal_view.ml b/app/journal_view.ml index 38e42c2..3f87ee1 100644 --- a/app/journal_view.ml +++ b/app/journal_view.ml @@ -853,7 +853,6 @@ module View = struct ?accessibility_identifier ?label ?attachments:(Option.map (fun view -> view.mount) attachments) - ~attachments_height:112 ?feedback:(Option.map (fun view -> view.mount) feedback) ~placeholder ~text:(Text_editing.Value.text value) diff --git a/apple-tests/amplify/JournalAmplifyAcceptance.swift b/apple-tests/amplify/JournalAmplifyAcceptance.swift deleted file mode 100644 index 3fca03c..0000000 --- a/apple-tests/amplify/JournalAmplifyAcceptance.swift +++ /dev/null @@ -1,72 +0,0 @@ -import Amplify -import AWSCognitoAuthPlugin -import LUIAppleBackend -import SwiftUI - -@MainActor private final class HubAcceptanceAuth: JournalAuthCapability { - var lookups = 0 - func currentUserID() async throws -> String? { lookups += 1; return nil } - func freshIDToken() async throws -> String { throw CancellationError() } - func signOut() async throws { throw CancellationError() } -} - -struct JournalAmplifyAcceptance: View { - @State private var result = "Waiting for configuration" - @State private var platform: JournalApplicationPlatform - private let auth: HubAcceptanceAuth - - init() { - let auth = HubAcceptanceAuth() - self.auth = auth - _platform = State(initialValue: JournalApplicationPlatform(services: JournalPlatformServices( - auth: auth, account: JournalAccountStore(load: { nil }, save: { _ in }, clear: {}), - managedSyncOrigin: "https://example.invalid"))) - } - - var body: some View { - VStack { - Text(result).padding() - Button("Test SDK Hub callbacks") { Task { await testHubCallbacks() } } - // Boots the embedded journal app through the LUI host; the platform - // bridge attaches inside JournalRuntime.start(). - JournalRuntimeHost( - platform: platform, - payload: (try? JournalNativeServices.startupPayload()) ?? Data(), - extensions: (try? JournalExtensions.registry()) ?? LUIAppleExtensionRegistry()) - .environment(\.scenePhase, .active) - .frame(height: 80) - }.task { - do { - // No user lookup, token retrieval or credential mutation is requested. - try JournalAmplifySession.configure() - try JournalAmplifySession.configure() - guard try Amplify.Auth.getPlugin(for: "awsCognitoAuthPlugin") is AWSCognitoAuthPlugin else { - result = "FAIL: Cognito plugin is unavailable" - return - } - result = "PASS: Real Cognito plugin configured; repeated configuration is safe" - } catch { - result = "FAIL: Configuration failed" - } - } - } - - @MainActor private func testHubCallbacks() async { - for name in [HubPayload.EventName.Auth.signedIn, HubPayload.EventName.Auth.signedOut, - HubPayload.EventName.Auth.sessionExpired, HubPayload.EventName.Auth.userDeleted] { - let before = auth.lookups - await Task.detached { - Amplify.Hub.dispatch(to: .auth, payload: HubPayload(eventName: name)) - }.value - let deadline = ContinuousClock.now.advanced(by: .seconds(3)) - while auth.lookups == before && ContinuousClock.now < deadline { - try? await Task.sleep(for: .milliseconds(10)) - } - guard auth.lookups > before, platform.authenticationRequired else { - result = "FAIL: SDK Hub callback did not reach the native auth owner" - return - } - } - result = "PASS: All four SDK Hub callbacks safely refresh native authentication" - } -} diff --git a/apple-tests/amplify/hub_fixture.ml b/apple-tests/amplify/hub_fixture.ml deleted file mode 100644 index b23f2fb..0000000 --- a/apple-tests/amplify/hub_fixture.ml +++ /dev/null @@ -1,105 +0,0 @@ -(* Headless fixture for the Amplify hub-callback acceptance test: a static - view driven by a Lui_app reducer app, replacing the previous Bonsai - computation. Dropped into a generated host as app/application.ml (see - tool/test_swiftui_amplify.py). *) - -open Lui_protocol -open Lui_elements - -type model = unit -type action = Nop - -let reducer () Nop = () - -let view _context _model _send = - column ~gap:16 ~padding:16 [ text ~value:"Native Hub callback acceptance" [] ] -;; - -(* --- headless host bridge ------------------------------------------------ *) - -let latest_patch = ref "" -let current_app : (model, action) Lui_app.reducer_app option ref = ref None - -let operating_system = function - | 1 -> MacOS - | 2 -> IOS - | 3 -> AndroidOS - | 4 -> LinuxOS - | 5 -> WindowsOS - | _ -> GenericOS -;; - -let host_kind = function - | 1 -> WebHost - | 2 -> SwiftUIHost - | 3 -> FlutterHost - | _ -> GenericHost -;; - -let backend profile = - { backend_profile = profile - ; apply_batch = - (fun batch -> - latest_patch := Lui_wire.encode_batch batch; - true) - } -;; - -let app () = - match !current_app with - | Some value -> value - | None -> invalid_arg "Hub acceptance fixture is not started" -;; - -let init platform_code host_code _payload = - latest_patch := ""; - let value = - Lui_app.create - (backend (profile (operating_system platform_code) (host_kind host_code))) - () - reducer - view - in - current_app := Some value; - ignore (Lui_app.start value); - ignore (Lui_app.flush value); - !latest_patch -;; - -let dispatch event = - latest_patch := ""; - ignore (Lui_app.dispatch_event (app ()) event); - ignore (Lui_app.flush (app ())); - !latest_patch -;; - -let extension_event _node _name _payload = "" - -let pump () = - latest_patch := ""; - ignore (Lui_app.flush (app ())); - !latest_patch -;; - -let dispose () = - latest_patch := ""; - Option.iter (fun value -> ignore (Lui_app.dispose value)) !current_app; - current_app := None; - !latest_patch -;; - -let root_node () = Lui_app.root_node (app ()) - -let native_hooks : Journal_bridge.hooks = - { init - ; dispatch - ; extension_event - ; pump - ; platform_event = (fun _ -> ()) - ; platform_response = (fun _ -> ()) - ; dispose - ; root_node - } -;; - -let () = Journal_bridge.register native_hooks diff --git a/apple-tests/authentication/JournalAuthenticationProbe.swift b/apple-tests/authentication/JournalAuthenticationProbe.swift index 554a19d..e7cdf5e 100644 --- a/apple-tests/authentication/JournalAuthenticationProbe.swift +++ b/apple-tests/authentication/JournalAuthenticationProbe.swift @@ -3,33 +3,13 @@ import SwiftUI /// The real presentation and owner run with a local provider; no account is accessed. @MainActor private struct ProbeAuthentication: JournalAuthenticationAPI { - struct Failure: LocalizedError { - var errorDescription: String? { "The verification code is incorrect. Try again." } - } let scenario: String - func perform(_ command: JournalAuthenticationCommand) async throws -> JournalAuthenticationStep { - switch command { - case .signIn: - try await Task.sleep(for: .milliseconds(300)) - switch scenario { - case "email": - return .challenge(.init(title: "Set up email verification", prompt: "Email address", input: .emailAddress)) - case "new-password": - return .challenge(.init(title: "Choose a new password", prompt: "New password", input: .newPassword)) - case "methods": - return .challenge(.init(title: "Choose a verification method", prompt: "Method", choices: ["SMS_MFA", "SOFTWARE_TOKEN_MFA"])) - default: - return .challenge(.init(title: "Verify your sign-in", prompt: "Verification code", input: .oneTimeCode, - detail: "Code sent to a***@example.com")) - } - case .confirm(let response): - if scenario == "code" && response != "123456" { throw Failure() } - return .complete - case .requestReset: return .resetConfirmation("a***@example.com") - case .confirmReset, .confirmAccount: return .signIn - case .resendAccount: return .confirmAccount("a***@example.com") - } + func signIn() async throws { + try await Task.sleep(for: .milliseconds(scenario == "busy" ? 15000 : 300)) + if scenario == "cancel" { throw CancellationError() } + if scenario == "error" { throw URLError(.notConnectedToInternet) } } + func cancelSignIn() { } } @main struct AuthenticationProbe: App { diff --git a/apple-tests/authentication/JournalAuthenticationTests.swift b/apple-tests/authentication/JournalAuthenticationTests.swift index 5cedf4b..0a7a5c1 100644 --- a/apple-tests/authentication/JournalAuthenticationTests.swift +++ b/apple-tests/authentication/JournalAuthenticationTests.swift @@ -1,87 +1,33 @@ import Foundation - @MainActor private final class API: JournalAuthenticationAPI { - var commands: [JournalAuthenticationCommand] = [] - var next = JournalAuthenticationStep.complete - var fail = false - var hold = false - var continuation: CheckedContinuation? - func perform(_ command: JournalAuthenticationCommand) async throws -> JournalAuthenticationStep { - commands.append(command) - if hold { return try await withCheckedThrowingContinuation { continuation = $0 } } - if fail { throw NSError(domain: "Authentication failed", code: 1) } - return next + var calls = 0, cancellations = 0 + var failure: Error? + var hold: CheckedContinuation? + var suspended = false + func signIn() async throws { + calls += 1 + if suspended { try await withCheckedThrowingContinuation { hold = $0 } } + if let failure { throw failure } } + func cancelSignIn() { cancellations += 1 } } - @main struct JournalAuthenticationTests { - static func check(_ value: Bool, _ message: String) throws { - if !value { throw NSError(domain: message, code: 1) } - } @MainActor static func main() async throws { - let api = API() + let api = API(), owner = JournalAuthentication(api: API()) + precondition(owner.canSubmit, "No credentials required") let model = JournalAuthentication(api: api) - try check(!model.canSubmit, "empty sign-in accepted") - await model.submit() - try check(api.commands.isEmpty, "empty form reached provider") - model.username = " alice " - model.password = "secret" - try check(model.canSubmit, "valid sign-in disabled") - api.fail = true - await model.submit() - try check(model.step == .signIn && model.password == "secret" && model.message != nil && !model.busy, - "sign-in failure lost retry state") - api.fail = false - let challenge = JournalAuthenticationStep.challenge(.init(title: "Verification", prompt: "Code")) - api.next = challenge - await model.submit() - try check(api.commands.last == .signIn(username: "alice", password: "secret"), "credentials changed") - try check(model.step == challenge && model.password.isEmpty && !model.canSubmit, "challenge transition retained password") - model.confirmation = "123456" - api.next = .complete - await model.submit() - try check(api.commands.last == .confirm("123456") && model.step == .complete && model.confirmation.isEmpty, - "challenge did not finish or retained its code") - print("PASS sign-in, failure retry, challenge and secret cleanup") - - model.back() - model.username = "alice" - model.forgotPassword() - try check(model.step == .resetRequest && model.canSubmit, "reset request inaccessible") - api.next = .resetConfirmation("a***@example.com") - await model.submit() - try check(api.commands.last == .requestReset("alice") && !model.canSubmit, "reset request changed") - model.confirmation = "654321" - model.newPassword = "replacement" - api.next = .signIn - await model.submit() - try check(api.commands.last == .confirmReset(username: "alice", code: "654321", password: "replacement"), "reset values lost") - try check(model.step == .signIn && model.newPassword.isEmpty && model.confirmation.isEmpty, "reset did not clear secrets") - print("PASS password reset") - - model.password = "secret" - api.next = .confirmAccount("email") - await model.submit() - await model.resend() - try check(api.commands.last == .resendAccount("alice"), "account code resend unavailable") - model.confirmation = "123456" - api.next = .signIn - await model.submit() - try check(api.commands.last == .confirmAccount(username: "alice", code: "123456"), "account confirmation changed") - print("PASS account confirmation and resend") - - model.password = "secret" - api.hold = true - let pending = Task { await model.submit() } - while api.continuation == nil { await Task.yield() } - let count = api.commands.count - await model.submit() - model.forgotPassword() - try check(api.commands.count == count && model.busy && model.step == .signIn, "busy form launched concurrent auth") - model.cancel() - api.continuation?.resume(returning: .complete) - await pending.value - try check(model.step == .signIn && !model.busy && model.password.isEmpty, "retired auth completion changed the form") - print("PASS duplicate submission and retired completion fencing") + api.failure = URLError(.notConnectedToInternet); await model.submit() + precondition(model.message != nil && model.canSubmit && !model.complete) + api.failure = CancellationError(); await model.submit() + precondition(model.message == nil && model.canSubmit && !model.complete) + api.failure = nil; api.suspended = true + let old = Task { await model.submit() } + while api.hold == nil { await Task.yield() } + await model.submit(); precondition(api.calls == 3 && model.busy) + model.cancel(); api.hold!.resume(); await old.value + precondition(!model.complete && !model.busy && api.cancellations == 1) + api.suspended = false; await model.submit() + precondition(model.complete && !model.canSubmit) + print("PASS credential-free presentation, error retry, cancel, duplicate and retired completion") } } diff --git a/apple-tests/authentication/JournalCognitoTests.swift b/apple-tests/authentication/JournalCognitoTests.swift new file mode 100644 index 0000000..80cfaa6 --- /dev/null +++ b/apple-tests/authentication/JournalCognitoTests.swift @@ -0,0 +1,182 @@ +import Foundation +import Security + +// Generates an independent RS256 issuer. All credentials below are synthetic. +@MainActor private final class Issuer { + let key: SecKey + let jwks: Data + var now = Date(timeIntervalSince1970: 1_800_000_000) + var stored: JournalCognitoTokens? + var requests: [URLRequest] = [] + var authorization: URL? + var callbackOverride: String? + var browserError: Error? + var tokenError: String? + var status = 200 + var tokenHold: CheckedContinuation? + var holdTokens = false + var browserHold: CheckedContinuation? + var holdBrowser = false + var defect: String? + var lifetime: Double = 3600 + var omitRefresh = false + var cancellations = 0 + var saveHold: CheckedContinuation? + var holdSave = false + let config = JournalCognitoConfiguration(domain: "auth.example.com", clientID: "public-client", + userPoolID: "test-pool", region: "us-east-1", redirectURI: "logseqjournal://auth/callback") + init() throws { + key = SecKeyCreateRandomKey([kSecAttrKeyType: kSecAttrKeyTypeRSA, kSecAttrKeySizeInBits: 2048] as CFDictionary, nil)! + var bytes = Array(SecKeyCopyExternalRepresentation(SecKeyCopyPublicKey(key)!, nil)! as Data), index = 0 + func read() -> [UInt8] { + index += 1; var count = Int(bytes[index]); index += 1 + if count > 127 { + let width = count & 127; count = 0 + for _ in 0.. String { + data.base64EncodedString().replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_").replacingOccurrences(of: "=", with: "") + } + func query(_ url: URL) -> [String: String] { + Dictionary(uniqueKeysWithValues: URLComponents(url: url, resolvingAgainstBaseURL: false)! + .queryItems!.map { ($0.name, $0.value ?? "") }) + } + func token(_ use: String) throws -> String { + var claims: [String: Any] = ["iss": defect == "issuer" ? "https://evil.example" : config.issuer, + "sub": "fixture-user", "exp": now.addingTimeInterval(lifetime).timeIntervalSince1970, + "iat": now.timeIntervalSince1970, "token_use": use] + if use == "id" { + claims["aud"] = defect == "audience" ? "other" : config.clientID + claims["nonce"] = defect == "nonce" ? "wrong" : query(authorization!)["nonce"] + } else { claims["client_id"] = config.clientID } + let input = Self.url64(try JSONSerialization.data(withJSONObject: ["alg": "RS256", "kid": "test-key"])) + + "." + Self.url64(try JSONSerialization.data(withJSONObject: claims)) + var signature = SecKeyCreateSignature(key, .rsaSignatureMessagePKCS1v15SHA256, Data(input.utf8) as CFData, nil)! as Data + if defect == "signature" { signature[0] ^= 1 } + return input + "." + Self.url64(signature) + } + func callback() -> URL { + URL(string: callbackOverride ?? "logseqjournal://auth/callback?state=\(query(authorization!)["state"]!)&code=synthetic-code")! + } + func session() -> JournalCognitoSession { + JournalCognitoSession(configuration: config, + storage: JournalCognitoStorage(load: { self.stored }, save: { value in + if self.holdSave { await withCheckedContinuation { self.saveHold = $0 }; self.holdSave = false } + self.stored = value + }, clear: { self.stored = nil }), + browser: JournalCognitoBrowser(authenticate: { url, _ in + self.authorization = url + if let error = self.browserError { throw error } + if self.holdBrowser { return try await withCheckedThrowingContinuation { self.browserHold = $0 } } + return self.callback() + }, cancel: { self.cancellations += 1 }), transport: { request in + self.requests.append(request) + var data = Data() + var status = 200 + if request.url!.path.hasSuffix("jwks.json") { data = self.jwks } + else if request.url!.path == "/oauth2/token" { + if self.holdTokens { await withCheckedContinuation { self.tokenHold = $0 } } + status = self.status + if let error = self.tokenError { + data = try JSONSerialization.data(withJSONObject: ["error": error, "error_description": "never display server secrets"]) + } else { + var object: [String: Any] = ["id_token": try self.token("id"), "access_token": try self.token("access"), + "expires_in": Int(self.lifetime), "token_type": "Bearer"] + if !self.omitRefresh { object["refresh_token"] = "synthetic-refresh+&=" } + data = try JSONSerialization.data(withJSONObject: object) + } + } + return (data, HTTPURLResponse(url: request.url!, statusCode: status, httpVersion: nil, headerFields: nil)!) + }, now: { self.now }) + } + var tokenRequests: [URLRequest] { requests.filter { $0.url?.path == "/oauth2/token" } } +} +@main struct JournalCognitoTests { + @MainActor static func rejects(_ operation: () async throws -> Void) async throws { + do { try await operation(); throw NSError(domain: "Expected rejection", code: 99) } + catch let error as NSError where error.domain == "Expected rejection" { throw error } + catch { } + } + @MainActor static func main() async throws { + let issuer = try Issuer(), session = issuer.session(); try await session.signIn() + let q = issuer.query(issuer.authorization!) + precondition(q["response_type"] == "code" && q["code_challenge_method"] == "S256" && q["scope"] == "openid") + precondition(q["state"]!.count >= 32 && q["nonce"]!.count >= 32) + let body = String(data: issuer.tokenRequests[0].httpBody!, encoding: .utf8)! + precondition(body.contains("code_verifier=") && !body.contains("client_secret") && !body.contains("password")) + let user = try await session.currentUserID(), id = try await session.freshIDToken() + precondition(user == "fixture-user" && id == issuer.stored!.idToken && issuer.tokenRequests.count == 1) + precondition(JournalCognitoOAuth.challenge("dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk") == "E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM") + print("PASS code/PKCE/state/nonce, signed identity and cached ID token") + for callback in ["other://auth/callback", "logseqjournal://other/callback", "logseqjournal://auth/wrong", + "logseqjournal://auth/callback?state=wrong&code=x", "logseqjournal://auth/callback?code=x", + "logseqjournal://auth/callback?state=x&state=x&code=y", "logseqjournal://auth/callback#code=x", + "logseqjournal://user@auth/callback?code=x", "logseqjournal://auth:123/callback?code=x"] { + let f = try Issuer(); f.callbackOverride = callback; let s = f.session() + try await rejects { try await s.signIn() }; precondition(f.tokenRequests.isEmpty && f.stored == nil) + } + for mode in ["cancel", "error", "expire", "replay", "signout", "duplicate", "mixed"] { + let f = try Issuer(); f.holdBrowser = true; let s = f.session() + let old = Task { try await s.signIn() }; while f.browserHold == nil { await Task.yield() } + let oldCallback = f.callback(), state = f.query(f.authorization!)["state"]! + if mode == "cancel" || mode == "replay" { s.cancelSignIn() } + if mode == "signout" { try await s.signOut() } + if mode == "expire" { f.now = f.now.addingTimeInterval(301) } + var callback = oldCallback + if mode == "error" { callback = URL(string: "logseqjournal://auth/callback?state=\(state)&error=access_denied")! } + if mode == "duplicate" { callback = URL(string: "logseqjournal://auth/callback?state=\(state)&code=x&code=y")! } + if mode == "mixed" { callback = URL(string: "logseqjournal://auth/callback?state=\(state)&code=x&error=access_denied")! } + f.browserHold!.resume(returning: callback); try await rejects { try await old.value } + precondition(f.stored == nil && f.tokenRequests.isEmpty); f.holdBrowser = false + if mode == "replay" { f.callbackOverride = oldCallback.absoluteString; try await rejects { try await s.signIn() } } + else { try await s.signIn(); precondition(f.stored != nil) } + } + print("PASS callback mismatch, duplicate/mixed parameters, cancel/error/expiry/replay/sign-out and retry") + for mode in ["nonce", "audience", "issuer", "signature", "expired", "refresh"] { + let f = try Issuer(); f.defect = mode + if mode == "expired" { f.lifetime = -1 }; if mode == "refresh" { f.omitRefresh = true } + let s = f.session(); try await rejects { try await s.signIn() }; precondition(f.stored == nil) + } + print("PASS invalid signature/issuer/audience/nonce/expiry/incomplete tokens") + let restored = issuer.session(); let restoredUser = try await restored.currentUserID(); precondition(restoredUser == user) + issuer.now = issuer.now.addingTimeInterval(3550); issuer.holdTokens = true; issuer.omitRefresh = true + let first = Task { try await restored.freshIDToken() }; while issuer.tokenHold == nil { await Task.yield() } + let second = Task { try await restored.freshIDToken() }; for _ in 0..<10 { await Task.yield() } + precondition(issuer.tokenRequests.count == 2); issuer.tokenHold!.resume() + let a = try await first.value, b = try await second.value + precondition(a == b && issuer.stored!.refreshToken == "synthetic-refresh+&=") + precondition(String(data: issuer.tokenRequests.last!.httpBody!, encoding: .utf8)!.contains("%2B%26%3D")) + print("PASS cold restore, singleflight concurrent refresh, retained token and form escaping") + issuer.now = issuer.now.addingTimeInterval(3550); issuer.holdTokens = false + issuer.status = 503; issuer.tokenError = "server_error" + try await rejects { _ = try await restored.freshIDToken() }; precondition(issuer.stored != nil) + issuer.status = 200; issuer.tokenError = nil; _ = try await restored.freshIDToken() + issuer.now = issuer.now.addingTimeInterval(3550); issuer.status = 400; issuer.tokenError = "invalid_grant" + try await rejects { _ = try await restored.freshIDToken() }; precondition(issuer.stored == nil) + let expiredUser = try await restored.currentUserID(); precondition(expiredUser == nil) + print("PASS transient refresh retry/preservation and invalid-grant expiry") + let retired = try Issuer(), rs = retired.session(); try await rs.signIn() + retired.now = retired.now.addingTimeInterval(3550); retired.holdTokens = true + let late = Task { try await rs.freshIDToken() }; while retired.tokenHold == nil { await Task.yield() } + try await rs.signOut(); retired.tokenHold!.resume(); try await rejects { _ = try await late.value } + let signedOut = try await rs.currentUserID(); precondition(retired.stored == nil && signedOut == nil) + print("PASS sign-out cleanup and late refresh isolation") + let saving = try Issuer(), ss = saving.session(); try await ss.signIn() + let previous = saving.stored!.idToken; saving.holdSave = true + let cold = saving.session() + let cancelledSave = Task { try await cold.signIn() } + while saving.saveHold == nil { await Task.yield() } + cold.cancelSignIn(); saving.saveHold!.resume() + try await rejects { try await cancelledSave.value } + precondition(saving.stored!.idToken == previous) + print("PASS cancellation during secure save preserves previous account") + } +} diff --git a/apple-tests/authentication/README.md b/apple-tests/authentication/README.md index 42d55cb..bfd70f6 100644 --- a/apple-tests/authentication/README.md +++ b/apple-tests/authentication/README.md @@ -1,80 +1,60 @@ -# Native authentication probe - -`python3 tool/build_authentication_probe.py` builds a disposable arm64 iPhone -Simulator application and prints its path and source hashes. Install that app -on an iOS 26 or later Simulator. It uses the production -`JournalAuthenticationView` and `JournalAuthentication` with a local API that -never accesses an account or the network. The sheet uses the production form -sizing, large detent, navigation container and Close placement. - -Launch `org.logseq.journal.authentication-probe` with `--scenario code`, -`--scenario email`, `--scenario new-password` or `--scenario methods`. -Optional `--large-text`, `--dark` and `--rtl` arguments select presentation -conditions in the probe only. Relaunch between scenarios to reset its state. - -Use only dummy values, for example username `probe` and password `sample`. -Any nonempty credentials advance to the selected challenge. In the code -scenario, `123456` completes; other responses produce a retryable local error. -Other scenarios complete with any nonempty response or a selected method. -Forgot password exercises the existing reset-code and new-password fields. - -Inspect these native behaviors through the Simulator UI: - -- Username starts focused; Next moves to the secure password field and Go - submits. Busy state disables duplicate submission. -- The code challenge focuses its code field, retains an incorrect code for - correction, displays the failure, and accepts a corrected code. -- Email setup offers an email keyboard; new-password challenges conceal text - and carry new-password content semantics. Custom responses remain ordinary - text. Method selection uses the native Picker. -- Close and swipe dismissal return to Authentication closed. Completion returns - to Authentication completed. Neither sends any real authentication request. -- With the keyboard visible, large text, dark appearance and RTL, inspect - readable messages and reachable fields, submission and dismissal. Repeat in - portrait and landscape. Record actual observed results and screenshots. - -This probe is visual acceptance infrastructure, not a duplicate authentication -state regression suite. `tool/test_swiftui_authentication.py` exercises the -presentation owner's commands, retry, cancellation and stale completions. The -probe does not validate Amplify service behavior, full-app graph lifecycle, -physical-device performance or automatic delivery of real verification codes. - -## Reset focus regression - -`reset-focus.cua.js` runs in the CUA JavaScript session, using only the documented -native application API. Build and launch the probe with `--scenario methods`. -Enter dummy credentials, submit, choose Back to sign in, then Forgot password?. -Run the script while the retained username is populated on Reset your password. -It requires the keyboard Go action to open code entry, Next to focus the secure -new-password field, and Go to return to sign in. It enters only local test data. - -The production owner of this defect is JournalAuthenticationView's FocusState -and view lifecycle. The public JournalAuthentication actions and completions -already pass the reset command sequence, but have no native responder or return -key state; they cannot reproduce a stale keyboard targeting a retired view. -The regression therefore runs at the actual native view boundary. It does not -duplicate command, persistence or transport tests, bypass an interface, or -inject an incorrect provider result. The same CUA check failed before the focus -change (Next remained visible on the reset request) and passed afterward. - - -## Native XCTest accessibility acceptance - -The standardization report's `batch16-authentication-acceptance.swift` and -`batch16-authentication-project.rb` are a replayable, disposable Xcode UI runner. -Copy them into a temporary directory as `DeviceAcceptance.swift` and `create.rb`, -run the Ruby generator with the xcodeproj gem, then run its DeviceAcceptance -scheme against the booted iPhone 13 Simulator after building/installing the probe. -No application target, production project or Dune change is required. - -The runner audits contrast, hit regions, descriptions and clipping on the real -view, exercises a local incorrect-code response, a full sheet-dismissal drag, -and large dark RTL scrolling and Close across rotation. Only disabled-control -contrast findings are exempt; enabled-content findings are retained. Native -color/lifecycle rendering is outside the pure authentication command owner's -state/effect boundary, so these checks do not duplicate that regression suite. -The light contrast regression fails before the semantic-color fix and passes -after it. Landscape contrast findings remain inconclusive: some exported crops -do not contain the target text, and sampled Close/Back colors contradict their -reported failure. Preserve the report's evidence and do not label the complete -matrix passing merely because the focused light regression passes. +# Browser authentication acceptance + +Current source reference: logseq/chat `a136ba3fc02d3dd717cb5f097686f29ce2bfb36b`, +`apple/Sources/LogseqChat/CognitoOAuth.swift` and `CognitoAuthProvider.swift`. +Journal uses ASWebAuthenticationSession, code + S256 PKCE, state and OIDC nonce. +It requests only `openid`. Account recovery and configured challenges belong to +Cognito's web page. The native app collects no credentials. + +Run `python3 tool/test_swiftui_authentication.py` for presentation owner actions, +then `python3 tool/test_swiftui_cognito.py` for the public session owner against an +independent synthetic RS256 issuer. These tests never access production +Keychain, real accounts or GUI. Tests cover exact callback identity, ambiguous +parameters, state/replay/expiry, JWT signature/issuer/audience/nonce, cold restore, +ID-token expiry, refresh singleflight and retries, invalid-grant retirement, +sign-out and stale completion fencing, including cancellation during secure save. +`python3 tool/test_swiftui_services.py` retains account/graph protocol and storage +threading checks. `python3 tool/test_swiftui_events.py` retains lifecycle checks. + +`python3 tool/build_authentication_probe.py` builds a disposable Simulator +application with the production view and a local presentation provider. Scenarios +are `success`, `error`, `cancel` and `busy`. The optional `--large-text`, `--dark` and +`--rtl` launch arguments retain visual checks. It does not open a real browser or +validate Cognito. Coordinate GUI ownership before installing or launching it. +The Continue button, progress, retry, Cancel, Close and sheet dismissal should be +reachable in portrait/landscape with large text. No username/password fields +remain. Do not label unobserved visual acceptance as passed. + +## Real Cognito prerequisite + +Pool `us-east-1_dtagLnju8`, client `69cs1lgme7p8kbgld8n5kseii6`, domain +`logseq-prod.auth.us-east-1.amazoncognito.com` match the existing Journal client +and current Chat config. A user/admin must confirm the public client has no +secret, supports authorization-code OAuth and Cognito managed/hosted login, +allows `openid`, and has exact callback `logseqjournal://auth/callback` registered. +No additional scope, logout redirect or AWS credentials are needed by this code. +The build script registers `logseqjournal` in generated host Info.plist before +signing. Registration on Cognito has not been verified or changed by this task. + +After secure handoff, the user enters credentials directly into the system +browser and handles the OS sign-in prompt. Verify success, web account recovery, +user cancel, offline retry, relaunch and sign-out, and confirm existing local +Graph/E2EE account scope through public application behavior. Do not record +tokens, passwords or authorization codes, install on a physical device, or open +personal graphs without separately authorized acceptance scope. + +## Storage and upgrade + +OAuth tokens use the same app Keychain access identity with a dedicated generic +password item `com.logseq.journal.cognito.tokens` / `current-session`, Data +Protection Keychain, non-synchronizable, AfterFirstUnlockThisDeviceOnly. Existing +local-account-binding and E2EE key queries remain unchanged. Existing Amplify +sessions require one new web sign-in; no SDK fallback decodes old credentials. +Successful OAuth commit retires the exact old SDK session item; explicit sign-out +also clears it. All blocking storage calls serialize outside MainActor. A cancelled +secure save rolls back its own write without overwriting a newer account. +Sign-out clears local OAuth credentials before bounded best-effort remote +revocation. Ephemeral browser sessions avoid retaining Cognito browser cookies. +Transient refresh failure retains stored credentials; invalid_grant clears the +session and reopens the existing authentication gate. Revocation of already +issued JWTs remains enforced by Cognito/the existing API security boundary. diff --git a/apple-tests/authentication/reset-focus.cua.js b/apple-tests/authentication/reset-focus.cua.js deleted file mode 100644 index 3ee884d..0000000 --- a/apple-tests/authentication/reset-focus.cua.js +++ /dev/null @@ -1,29 +0,0 @@ -// Run in cua_repl against the local AuthenticationProbe. -// Precondition: use the UI to sign in with dummy values, return to sign in, -// then choose Forgot password? with the retained username still populated. -// This checks native focus and keyboard submission, not authentication decisions. -var authenticationSimulator = await cua.getApp("com.apple.iphonesimulator"); -async function authenticationState() { - return await authenticationSimulator.getAXState({disableDiffing: true, emit: false}); -} -function authenticationElement(state, pattern) { - const line = state.split("\n").find(line => pattern.test(line)); - if (!line) throw new Error("Missing native element: " + pattern + "\n" + state); - return Number(line.trim().match(/^\d+/)[0]); -} -var resetState = await authenticationState(); -authenticationElement(resetState, /heading Description: Reset your password/); -await authenticationSimulator.click(authenticationElement(resetState, /button Description: go, ID: Go/)); -resetState = await authenticationState(); -await authenticationSimulator.setValue( - authenticationElement(resetState, /text field .*Value: Verification code/), "123456"); -resetState = await authenticationState(); -await authenticationSimulator.click(authenticationElement(resetState, /button Description: next, ID: Next:/)); -resetState = await authenticationState(); -await authenticationSimulator.click(authenticationElement(resetState, /button s$/)); -resetState = await authenticationState(); -authenticationElement(resetState, /secure text field .*Value: •.*Placeholder: New password/); -await authenticationSimulator.click(authenticationElement(resetState, /button Description: go, ID: Go/)); -resetState = await authenticationState(); -authenticationElement(resetState, /heading Description: Sign in to Logseq/); -nodeRepl.write("PASS native reset Go submission, code Next focus, secure input and password Go submission"); diff --git a/apple-tests/editor/composer_probe.ml b/apple-tests/editor/composer_probe.ml index 3b62f57..505860a 100644 --- a/apple-tests/editor/composer_probe.ml +++ b/apple-tests/editor/composer_probe.ml @@ -45,22 +45,6 @@ let view _context model_source send = let latest_patch = ref "" let current_app : (model, action) Lui_app.reducer_app option ref = ref None -let operating_system = function - | 1 -> MacOS - | 2 -> IOS - | 3 -> AndroidOS - | 4 -> LinuxOS - | 5 -> WindowsOS - | _ -> GenericOS -;; - -let host_kind = function - | 1 -> WebHost - | 2 -> SwiftUIHost - | 3 -> FlutterHost - | _ -> GenericHost -;; - let backend profile = { backend_profile = profile ; apply_batch = @@ -80,7 +64,7 @@ let init platform_code host_code _payload = latest_patch := ""; let value = Lui_app.create - (backend (profile (operating_system platform_code) (host_kind host_code))) + (backend (Lui_native_bridge.profile platform_code host_code)) initial reducer view diff --git a/apple-tests/native-events/JournalLUIEventsTests.swift b/apple-tests/native-events/JournalLUIEventsTests.swift index 384c5bc..1732463 100644 --- a/apple-tests/native-events/JournalLUIEventsTests.swift +++ b/apple-tests/native-events/JournalLUIEventsTests.swift @@ -65,6 +65,16 @@ private let receivePatch: PatchCallback = { source in (.appear(node: node), "appear"), (.press(node: node), "press"), (.longPress(node: node), "longPress"), + (.pressDetail(node: node, x: -0.25, y: 12.5, modifiers: 15, button: 2, + targetClass: "日记📓"), "pressDetail:-0.25:12.5:15:2:日记📓"), + (.pointerDown(node: node, x: 1.25, y: -2.5, modifiers: 4, button: 0, + targetClass: ""), "pointerDown:1.25:-2.5:4:0:"), + (.pointerUp(node: node, x: 0, y: 0, modifiers: 0, button: 1, + targetClass: "row"), "pointerUp:0:0:0:1:row"), + (.pointerEnter(node: node), "pointerEnter"), + (.pointerLeave(node: node), "pointerLeave"), + (.contextMenuPress(node: node, x: 3.5, y: 4.25, modifiers: 8, button: 2, + targetClass: "菜单"), "contextMenuPress:3.5:4.25:8:2:菜单"), (.textChanged(node: node, text: "日记\n\"draft\""), "textChanged:日记\n\"draft\""), (.submit(node: node), "submit"), (.dismiss(node: node), "dismiss"), @@ -133,6 +143,11 @@ private let receivePatch: PatchCallback = { source in try check(value == "platform:410042", "binary platform envelope changed") } for _ in 0..<100 { + for (event, _) in cases.prefix(9) { + patches.removeAll() + try check(JournalLUIEvents.dispatch(event) == 1 && patches.count == 1, + "repeated pointer dispatch failed") + } patches.removeAll() reenter = true try check(JournalLUIEvents.dispatch(.press(node: node)) == 1, "repeated dispatch failed") diff --git a/apple-tests/native-events/bridge_fixture.ml b/apple-tests/native-events/bridge_fixture.ml index 79ccf89..905f013 100644 --- a/apple-tests/native-events/bridge_fixture.ml +++ b/apple-tests/native-events/bridge_fixture.ml @@ -27,6 +27,11 @@ let patch node text = { generation = 1; ops = [ SetProp (node, TextValue, StringValue text) ] } ;; +let pointer_patch kind node (detail : pointer_detail) = + patch node (Printf.sprintf "%s:%g:%g:%d:%d:%s" kind detail.x detail.y + detail.modifiers detail.button detail.target_class) +;; + let dispatch = function | Appear node -> patch node "appear" | Press node -> patch node "press" @@ -44,6 +49,14 @@ let dispatch = function | VisibleRange (node, first, last) -> patch node (Printf.sprintf "visibleRange:%d:%d" first last) | Picked (node, payload) -> patch node ("picked:" ^ payload) + | PressModifiers (node, modifiers) -> patch node ("pressModifiers:" ^ string_of_int modifiers) + | PressDetail (node, detail) -> pointer_patch "pressDetail" node detail + | PointerDown (node, detail) -> pointer_patch "pointerDown" node detail + | PointerUp (node, detail) -> pointer_patch "pointerUp" node detail + | PointerEnter node -> patch node "pointerEnter" + | PointerLeave node -> patch node "pointerLeave" + | ContextMenuPress (node, detail) -> pointer_patch "contextMenuPress" node detail + | Load node -> patch node "load" | ExtensionEvent _ -> failwith "extension bypassed its host hook" ;; diff --git a/apple-tests/native-outline/outline_probe.ml b/apple-tests/native-outline/outline_probe.ml index c24235e..0141103 100644 --- a/apple-tests/native-outline/outline_probe.ml +++ b/apple-tests/native-outline/outline_probe.ml @@ -150,22 +150,6 @@ let view _context model_source send = let latest_patch = ref "" let current_app : (model, action) Lui_app.reducer_app option ref = ref None -let operating_system = function - | 1 -> MacOS - | 2 -> IOS - | 3 -> AndroidOS - | 4 -> LinuxOS - | 5 -> WindowsOS - | _ -> GenericOS -;; - -let host_kind = function - | 1 -> WebHost - | 2 -> SwiftUIHost - | 3 -> FlutterHost - | _ -> GenericHost -;; - let backend profile = { backend_profile = profile ; apply_batch = @@ -185,7 +169,7 @@ let init platform_code host_code _payload = latest_patch := ""; let value = Lui_app.create - (backend (profile (operating_system platform_code) (host_kind host_code))) + (backend (Lui_native_bridge.profile platform_code host_code)) initial reducer view diff --git a/apple-tests/platform-services/JournalAccountThreadingTests.swift b/apple-tests/platform-services/JournalAccountThreadingTests.swift index 60b1ebc..f47c314 100644 --- a/apple-tests/platform-services/JournalAccountThreadingTests.swift +++ b/apple-tests/platform-services/JournalAccountThreadingTests.swift @@ -51,7 +51,7 @@ enum JournalLocalAccountBindingStore { } } -@MainActor final class JournalAmplifySession: JournalAuthCapability { +@MainActor final class JournalCognitoSession: JournalAuthCapability { static var signOutCalls = 0 static var userCalls = 0 func currentUserID() async throws -> String? { Self.userCalls += 1; return "fixture-user" } @@ -59,6 +59,8 @@ enum JournalLocalAccountBindingStore { func signOut() async throws { Self.signOutCalls += 1 } } +@MainActor enum JournalCognitoNative { static let shared = JournalCognitoSession() } + @main struct JournalAccountThreadingTests { @MainActor static func waitFor(_ condition: () -> Bool) async throws { let deadline = ContinuousClock.now + .seconds(3) @@ -94,19 +96,19 @@ enum JournalLocalAccountBindingStore { let authentication = Task { try await platform.response(for: .authenticatedUser) } try await waitFor { JournalLocalAccountBindingStore.state.withLock { $0.operations == ["load"] } } let signOut = Task { try await platform.response(for: .signOut) } - try await waitFor { JournalAmplifySession.signOutCalls == 1 } + try await waitFor { JournalCognitoSession.signOutCalls == 1 } authGate.signal() try await canceled(authentication) let signedOut = try await signOut.value - precondition(signedOut == .signedOut && JournalAmplifySession.userCalls == 0) + precondition(signedOut == .signedOut && JournalCognitoSession.userCalls == 0) precondition(JournalLocalAccountBindingStore.state.withLock { $0.value == nil }) let saveGate = JournalLocalAccountBindingStore.hold("save") let saving = Task { try await platform.response(for: .authenticatedUser) } try await waitFor { JournalLocalAccountBindingStore.state.withLock { $0.operations.contains("save") } } let cleanup = Task { try await platform.response(for: .signOut) } - try await waitFor { JournalAmplifySession.signOutCalls == 2 } - // Cancellation must not undo cleanup after the SDK has confirmed sign-out. + try await waitFor { JournalCognitoSession.signOutCalls == 2 } + // Cancellation must not undo cleanup after the provider has confirmed sign-out. cleanup.cancel() let responsive = try await platform.response(for: .terminationReady) precondition(responsive == .terminationReady) diff --git a/docs/agent-guide/implemented/bugfix/2026-10-08-coalesced-asset-refresh.md b/docs/agent-guide/implemented/bugfix/2026-10-08-coalesced-asset-refresh.md deleted file mode 100644 index a4ce2ab..0000000 --- a/docs/agent-guide/implemented/bugfix/2026-10-08-coalesced-asset-refresh.md +++ /dev/null @@ -1,47 +0,0 @@ -# Coalesce asset refresh and defer hidden roots - -## Problem - -Every graph notification rereads all retained media roots, including hidden, empty and already pending roots, and restarts both offline scans. A small event or burst therefore multiplies asset queries, cancellation and staging work. The existing changefeed is a logical block/page/structure summary, not a complete reverse asset dependency index. - -## Decision - -Use the existing public runtime and policy owners to coalesce refresh demand. Retained hidden roots become dirty and discard global projection-bound cursors; root, asset and preview activation obtain a fresh first page. Each active root has at most one query in flight and one merged fresh demand. Superseded query completions cannot publish data or erase pending demand. Offline scans retain committed consumers while coalescing repeated changes through their current request and at most one required followup per reason. - -Ordinary asset policy/runtime interfaces distinguish graph-change invalidation from the existing same-configuration lifecycle Refresh. Same graph/day/settings Refresh is idempotent; an actual graph-change invalidation restarts Finished scans and merges pending demand. Application routes raw graph push to that ordinary invalidation entry. This routine implementation choice is within the authorized F2 fix; it introduces no new UX payload or protected interface. Keeping the distinction avoids both repeated lifecycle scans and a permanent same-settings no-op after an actual graph change. - -Existing block/structure changes and known referenced asset identities are positive relevance signals. Missing, empty, ancestor, schema and negative dependencies require a conservative fallback for active roots. Hidden history is never eagerly rescanned by that fallback. Resync is conservative. No protected spec, wire or Dune change is needed; existing APIs may remain conservative where they cannot prove exclusion. - -The current media owner has no complete dependency proof for any active group. This implementation therefore treats every active root as potentially relevant on raw projection notifications. It does not manufacture a precise exclusion rule from unmatched block/structure UUIDs; hidden groups remain lazy. Positive changefeed signals explain known relationships, while safe exclusion is intentionally not promised. - -## Alternatives considered - -### Match only changed root block UUIDs - -Public Database.listen/apply_authoritative probes show that child membership reports the child plus Children_interest(parent), while foreign referenced asset metadata reports only the asset UUID. The holder/root need not appear in block_uuids. - -### Add protected read-dependency and scoped-cursor interfaces - -The user questioned their necessity. Existing API conservative invalidation is sufficient to remove hidden and duplicate work without promising complete dependency knowledge. The proposal is withdrawn and no protected file is edited. - -## Acceptance criteria - -- Public production-owner tests first fail on hidden eager refresh and duplicate in-flight requests, then pass after the fix. -- Cross-page metadata, empty-to-populated roots, membership moves/deletes, preview ownership, resync, stale cursors and dirty in-flight completions preserve eventual freshness. -- Retained hidden roots issue no immediate graph-change query; activation obtains a fresh page. Burst demand does not issue duplicate in-flight work. -- Offline scan demand coalesces without retaining superseded staged results or dropping committed offline ownership. - -## Consequences - -- Change summaries do not expose every negative, schema or ancestor dependency. Active roots may refresh conservatively; this change does not claim perfect dependency-selective queries. -- Asset continuation cursors bind the entire projection version. All relevant graph changes invalidate saved cursors even if a root's current display appears unchanged. -- Only synthetic public API effect/query counters are evidence; no real graph/account/phone timings or physical bytes are claimed. -- Final public-owner validation passed 24 new cases: 19 demonstrated behavior RED before GREEN and 5 explicitly preserve compatible behavior. Media has 15 new passing cases plus its existing scenarios; Policy has 15 total cases (6 existing, 8 new pure-owner cases and one narrow adapter routing case). At 64 retained roots with 2 active roots, invalidation issues 2 reads, and 32 further notifications do not add in-flight requests. -- A clean Media continuation's typed stale cursor restarts once at cursor=None; a failed fresh first page stops visibly. Policy's clean stale failure remains Failed with committed demand retained, until graph invalidation or configuration replacement; no comprehensive automatic stale recovery is claimed. -- Application raw push and owned pull resync use the same invalidation. The pull binding checks the current Worker request map and protocol request UUID before removing ownership. This private Application side effect has source review, not an independent runtime test; public Runtime reset/late-response tests do not execute that binding. -- Final `dune build @all app/native_embed.exe.o` and `dune runtest --force` both exited 0. The latter used approved synthetic loopback permissions with module caches confined to the worktree. Protected spec/Dune diff is empty. - -## Questions - -- Authorization is resolved: the human authorized F1/F2/F3/F5 fixes and tests, and explicitly directed continuation of conservative F2 using existing APIs after withdrawing the protected interface proposal. -- Regression boundary is resolved: public media runtime and asset policy events/completions/state/effects own the amplification and receive regression tests. No duplicate runner/transport tests are added for behavior these owners reproduce. diff --git a/docs/agent-guide/implemented/feature/2026-09-08-journals-favorites-navigation.md b/docs/agent-guide/implemented/feature/2026-09-08-journals-favorites-navigation.md index 8b497eb..01d4357 100644 --- a/docs/agent-guide/implemented/feature/2026-09-08-journals-favorites-navigation.md +++ b/docs/agent-guide/implemented/feature/2026-09-08-journals-favorites-navigation.md @@ -324,10 +324,7 @@ behavior that pure tests cannot establish. ## Consequences Favorites reads fail explicitly above the 10,000-membership resource limit and do -not display child previews. Invalidation on every graph push is conservative: -large active lists may refresh for unrelated changes, while inactive lists only -become dirty. This keeps dependency tracking complete and subscription memory -constant. Graph replacement clears both destinations. Native adapters add a small +not display child previews. Graph replacement clears both destinations. Native adapters add a small host presentation boundary, verified by native widget tests; graph, cache, draft, and mutation ownership remains in OCaml. @@ -359,10 +356,7 @@ the public contract, verifies passing behavior, and checks the affected suites. The read implementation audit explicitly verifies link-only publication through public authoritative completion and snapshot reads. It asserts that the logical projection advances, the membership UUID is published, and an old cursor is -rejected. This passes with the existing graph publication owner. Favorites -conservatively invalidates on every logical graph push, including resync, so its -subscription footprint remains constant and covers filtered targets, ancestry, -restoration, and hidden-page discovery without a growing per-target watch list. +rejected. This passes with the existing graph publication owner. Native Capture uses an application-owned adapter around the existing Flutter FAB, persistent bottom sheet, and MessageComposer. The existing expandable composer @@ -385,8 +379,8 @@ keeps a rendered window of at most 128 rows. Journals and Favorites share the existing root service and row renderer. Favorites owns its lazy cache, refresh staging, request generations, and membership anchor. -Every graph push invalidates the cache; inactive reads remain isolated and graph -replacement discards both tab lifetimes. Capture draft/save state stays in the +Inactive reads remain isolated and graph replacement discards both tab lifetimes. +Capture draft/save state stays in the application, while its native adapter owns only editing controls and sheet focus. The public widget registry renders the OCaml NavigationBar descriptor using the diff --git a/docs/agent-guide/implemented/feature/2026-10-08-cognito-web-authentication.md b/docs/agent-guide/implemented/feature/2026-10-08-cognito-web-authentication.md new file mode 100644 index 0000000..d04f3a0 --- /dev/null +++ b/docs/agent-guide/implemented/feature/2026-10-08-cognito-web-authentication.md @@ -0,0 +1,40 @@ +# Cognito Web Authentication + +## Problem + +Journal collects passwords and depends on Amplify/AWS Swift SDK for Cognito sessions. The user requests the browser login used by current logseq/chat and removal of the SDK. + +## Decision + +Use ASWebAuthenticationSession with authorization code, S256 PKCE, state and nonce. Validate the exact callback and signed ID/access token claims. A serialized session owner provides existing currentUserID/freshIDToken/signOut capabilities, persisted Keychain tokens, singleflight refresh and stale-completion fences. Keep local account/E2EE storage and OCaml graph decisions unchanged. Request only openid. Remove native credential/challenge forms, SDK wrappers and package pins. No Dune/spec changes. + +Reference: logseq/chat a136ba3fc02d3dd717cb5f097686f29ce2bfb36b, apple/Sources/LogseqChat/CognitoOAuth.swift and CognitoAuthProvider.swift. Local Chat 37c034e4 is obsolete. Journal owner authorized isolated branch from b36bea83. Source owner owns Swift auth/session; OCaml reducers cannot reproduce browser or token transport behavior, so tests target this public Swift boundary. + +## Alternatives considered + +### Retain Amplify native authentication + +Conflicts with the user's explicit SDK removal. Embedded WebView and password grants are unnecessary and are excluded. + +## Implementation + +- Credential-free sign-in opens a system authentication session; cancel/error preserves account state and permits retry. +- Callback scheme/host/path/state/duplicates/expiry/replay are rejected before token transport. Concurrent refresh joins one request; sign-out retires in-flight work. +- Tokens use Keychain and validated issuer/audience/nonce/signature; account binding and E2EE contracts remain unchanged. +- No AWS SDK imports, wrapper or package pins remain. Swift and simulator compile, native owner tests and available repository checks pass. + +## Consequences + +- Existing SDK sessions require browser sign-in once; persisted local graph binding is retained. +- The same pool us-east-1_dtagLnju8, client 69cs1lgme7p8kbgld8n5kseii6 and domain logseq-prod.auth.us-east-1.amazoncognito.com are referenced by Chat. New logseqjournal://auth/callback must be confirmed registered and code flow enabled on the public no-secret client. No AWS configuration changes are authorized or performed. +- Synthetic tests cannot prove real Cognito sign-in, account recovery screens or system consent. Those need user secure handoff and GUI coordination. No device installation or personal graph access. + + +## Verification + +- Formal public Swift session tests pass with an independent synthetic RS256 issuer. Coverage includes code/PKCE/state/nonce, callback scheme/host/path/duplicates/expiry/replay, issuer/audience/signature rejection, cold restoration, cached ID token, refresh singleflight and form escaping, network retry, invalid-grant clearing, sign-out and stale completion isolation. Cancellation during secure save was reproduced failing and now preserves the previous account, including cold restore. +- Existing platform service/account threading and lifecycle event tests pass. Native synthetic Simulator probe visibly passed initial sign-in, retryable error, busy/Cancel, Close and success dismissal. It used no real account, remote authentication, Keychain, graph or device. +- Complete current production Swift sources compile and link for iOS Simulator with only the owner-approved read-only LUI ee51 static library and OCaml object, without AWS objects. macOS authentication modules compile. Full macOS product remains blocked by unchanged baseline JournalChrome.swift:232 listSectionSpacing unavailable in macOS; it was reported to the consumer owner and not adapted here. +- Repository dune build passes. Force runtest initially passed 19 suites/445 cases and source boundary. Loopback TLS peer bind was sandbox-denied and an external-build repository-root lookup failed. The same sync suite (194 cases) passed with loopback permitted; the public compiler boundary (1 case) passed from the repository root using its external build symlink. Total coverage: 21 suites/640 cases. No Dune/spec/upstream edits. +- SwiftPM dependency graph contains only local LUI and no remote pins. Package.resolved and all AWS transitive pins are removed. Existing CryptoKit/Security, local-account binding and E2EE files remain unchanged. Spec-dev-tool check and diff whitespace checks pass. +- Real Cognito login, web recovery/challenges, production Keychain operations and OS consent remain untested. The exact new callback registration has not been verified; backend changes and real-account interaction are not performed. diff --git a/docs/agent-guide/proposed/feature/2026-10-08-2026-10-08-upgrade-datascript-lui.md b/docs/agent-guide/proposed/feature/2026-10-08-2026-10-08-upgrade-datascript-lui.md new file mode 100644 index 0000000..951a6c2 --- /dev/null +++ b/docs/agent-guide/proposed/feature/2026-10-08-2026-10-08-upgrade-datascript-lui.md @@ -0,0 +1,32 @@ +# Upgrade Journal to current DataScript and LUI + +## Problem + +Journal uses an old LUI revision whose host protocol differs from current upstream. Latest Journal main pins DataScript before the new recursive-rule precheck fix. + +## Proposal + +Honor the explicit upgrade request: upgrade DataScript to 0561660e4894faee250d551ab2a32a6b5c25a5fb and upgrade LUI to 23b7563c21aa17e0f59da6527ee0127de8c1f0b6. Align installation manifests and boundary assertions, adapt public calls, and validate native behavior using disposable fixtures. Build dependencies in a task-local overlay using the existing read-only compiler switch. + +## Alternatives considered + +### Keep the old LUI revision + +Rejected because the user requested current upstream main and a completed adaptation. + +## Acceptance criteria + +- All applicable installation manifests and boundary assertions use the verified exact revisions. +- Full build and forced regression checks pass against those dependencies. +- Native navigation, Capture, sheets, menus, list updates, and media paths are checked in an isolated Simulator container; failures and limits are reported precisely. +- Independent review and a local commit are delivered. + +## Risks + +- Host enum, bridge lifetime, event and runtime changes can affect native behavior even when compilation succeeds. +- Historical UI issues must be distinguished from upgrade regressions. +- Dune and spec OCaml changes require separate explicit approval if found necessary. + +## Authorization + +The delegation explicitly requests implementation on current main, exact dependency SHAs, local commits, independent review, isolated Simulator testing, and no push, PR, merge, or phone installation. No unanswered design choice is required to begin this scope. diff --git a/docs/agent-guide/proposed/feature/2026-10-08-journal-composer-multiselect.md b/docs/agent-guide/proposed/feature/2026-10-08-journal-composer-multiselect.md new file mode 100644 index 0000000..8bab8bb --- /dev/null +++ b/docs/agent-guide/proposed/feature/2026-10-08-journal-composer-multiselect.md @@ -0,0 +1,51 @@ +# Journal Composer multiselect consumption + +## Problem + +The user requested Journal consume merged LUI PR159, including real attachment layout and multiselect. Journal currently forces a 112pt strip and accepts one native pick at a time; asynchronous pick completion must keep draft/request ownership, temporary resources, and durable import order. + +## Proposal + +Pin the verified merged main ee51c9747e584edd2ac09cc2cefee130622d8bda. Use the public default 128pt strip/120pt card. Enable ordered Files and Photos multiselect for staged Capture picks; retain single selection for detail imports. Deliver a batch with its original request identity, deduplicate by stable source identity, retain successful picks on partial staging failure, preserve cancellation, discard rejected/stale temporary copies, and retain Journal's existing nine-item limit. Preserve existing durable upload intent owner and import completion behavior; ensure captured attachment import admission stays ordered. + +## Alternatives considered + +### Only update the dependency pin + +Rejected because it leaves the production picker single-select and clips the new cards. + +### Copy Gallery send/clear and example limits + +Rejected because Journal owns durable staging/upload intents and already has its own attachment limit. + +## Consequences + +The native consumer and event decoder change together; tests must cover selection order, cancellation, duplicate/removed selections, partial failure and graph/draft fencing. No protected Dune/spec changes are planned. Existing postcommit Capture confirmation Full remains separate; any verification blocked by it will report the minimal additional scope instead of replaying a saved mutation. + +## Acceptance criteria + +- Exact manifest/lock/source pins agree with verified LUI main. +- Public tests cover batch order, cancellation, duplicate/removal, partial failure, capacity and stale resource ownership. +- Full affected regressions and native synthetic multiselect/layout verification are recorded with honest limits. +- Independent review and a local commit are delivered without Journal publication. + +## Risks + +- Async native selections can complete after graph/draft replacement and must not acquire a newer request identity. +- Durable staging intents must precede release of source copies; existing Capture Full may block complete UI Send acceptance. + +## Authorization + +The user explicitly asked to merge LUI PR159 and then adapt Journal to latest LUI. The delegated scope permits necessary Journal consumer implementation, local commits, independent review and isolated synthetic UI tests, with no Journal publishing/merge/phone installation or seven HIG fixes. PR159 merge was verified from GitHub and exact main fetched before pin update. No unanswered design question remains. + +## Verification status + +The batch ownership boundary was reproduced through public Root_navigation events before implementation. Import admission belongs to Application's effect queue, which the pure reducer cannot execute; its regression uses the existing public app_with_service hooks and real Concurrent Worker lanes. The unsequenced variant starts the second import before the first completes and fails the regression. The initial implementation also failed the public Error info visibility assertion; the final implementation passes both and keeps successful siblings after a failed item. + +Final full build and force runtest passed with 21 Alcotest suites / 640 cases, including source boundary and interface checks. The first transport run was blocked by sandbox loopback binding; the unchanged suite passed with local loopback permission. Native bridge 23 cases passed. Exact LUI main CI passed all six jobs; its local OCaml 87 / macOS Apple 167 cases and iOS Simulator static build passed. Independent review findings were resolved. The final production Swift host compiled without diagnostic substitutions. + +The native consumer subscribes through the public context.revision contract and keeps a transparent, noninteractive presentation anchor for the otherwise empty Capture adapter. The final production binary (source ca2fa064, SHA256 a799a5949076692fff0009d5c4dcb74b1b73ee9cfe5e4d568d737793ff7b80a8) was installed only in the isolated org.logseq.journal.composer-20261008 bundle on B15AA93D-9889-4D75-8A4A-ABC12F00B478. The user explicitly approved limited XCTest on synthetic data after CUA input failed. + +Actual native tests passed Files cancellation/reopening twice, mixed three-file selection, duplicate re-selection, removal/re-addition and discard; Photos selected the synthetic green image first and blue second, preserved this order, retained both after a second picker cancellation, and discarded them. Tests used measured visible coordinates where the system AX node was not hittable. A middle image card's close-center tap did not remove it; later removal passed with the target in the first card position. The image AX frame exceeds the 120pt card width, so the middle-card normal hit boundary remains an unconfirmed risk, not a confirmed upstream defect. + +One distinct batch Send was actually performed. It wrote a single queued text mutation (1054f7d6-3d95-4e34-8ad3-7048b47db657, attemptCount 0), collapsed Composer and showed Error info. The subsequent read-only SQLite snapshot had zero asset upload intents. The observation test passed, but this is not successful batch-import acceptance. Exact error detail was not captured; do not classify it as the previously proven Full condition or a DataScript regression. The saved mutation was not replayed. The fixture blocks auth and cannot validate server acknowledgment/upload. Nine-item overflow, horizontal scrolling, partial native transfer failure, graph switch while picking, and successful native batch import remain unverified. Keep this decision proposed. GUI and XCTest are stopped and released to the peer task. diff --git a/logseq_db_storage.opam b/logseq_db_storage.opam index 210dc03..356c41f 100644 --- a/logseq_db_storage.opam +++ b/logseq_db_storage.opam @@ -19,8 +19,8 @@ depends: [ "yojson" {= "3.0.0"} ] pin-depends: [ - ["datascript_ocaml.dev" "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d"] - ["datascript-ocaml-native.dev" "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d"] + ["datascript_ocaml.dev" "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb"] + ["datascript-ocaml-native.dev" "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb"] ["melange-edn-core.0.5.0" "git+https://github.com/logseq/melange-edn.git#main"] ["melange-edn-native.0.5.0" "git+https://github.com/logseq/melange-edn.git#main"] ["melange-transit-core.0.1.2" "git+https://github.com/logseq/melange-transit.git#main"] diff --git a/logseq_db_storage.opam.locked b/logseq_db_storage.opam.locked index bc08c39..0a2754a 100644 --- a/logseq_db_storage.opam.locked +++ b/logseq_db_storage.opam.locked @@ -69,11 +69,11 @@ build: ["dune" "build" "-p" name "-j" jobs] pin-depends: [ [ "datascript-ocaml-native.dev" - "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d" + "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb" ] [ "datascript_ocaml.dev" - "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d" + "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb" ] [ "melange-edn-core.0.5.0" diff --git a/logseq_db_worker.opam b/logseq_db_worker.opam index 0163631..bf4a041 100644 --- a/logseq_db_worker.opam +++ b/logseq_db_worker.opam @@ -23,8 +23,8 @@ depends: [ ] pin-depends: [ ["rrbvec.dev" "git+https://github.com/logseq/rrbvec.git#main"] - ["datascript_ocaml.dev" "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d"] - ["datascript-ocaml-native.dev" "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d"] + ["datascript_ocaml.dev" "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb"] + ["datascript-ocaml-native.dev" "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb"] ["melange-edn-core.0.5.0" "git+https://github.com/logseq/melange-edn.git#main"] ["melange-edn-native.0.5.0" "git+https://github.com/logseq/melange-edn.git#main"] ["melange-transit-core.0.1.2" "git+https://github.com/logseq/melange-transit.git#main"] diff --git a/logseq_db_worker.opam.locked b/logseq_db_worker.opam.locked index 4d07dab..5f27d91 100644 --- a/logseq_db_worker.opam.locked +++ b/logseq_db_worker.opam.locked @@ -191,11 +191,11 @@ pin-depends: [ ] [ "datascript-ocaml-native.dev" - "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d" + "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb" ] [ "datascript_ocaml.dev" - "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d" + "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb" ] [ "melange-edn-core.0.5.0" diff --git a/logseq_db_worker/lib/pure_reducer/core.ml b/logseq_db_worker/lib/pure_reducer/core.ml index ba71113..1602067 100644 --- a/logseq_db_worker/lib/pure_reducer/core.ml +++ b/logseq_db_worker/lib/pure_reducer/core.ml @@ -653,7 +653,7 @@ let step state event = | Some (_, transfer) -> transfer | None -> Transfer.create - (Transfer.config ~active:3 ~foreground_reserved:1 ~pending:128 ~retries:3 + (Transfer.config ~active:3 ~foreground_reserved:1 ~pending:128 ~retries:0 |> Result.get_ok) ~scope:context.scope ~online:(asset_online state) diff --git a/logseq_db_worker/test/test_pure_reducer.ml b/logseq_db_worker/test/test_pure_reducer.ml index 8684fcd..28bba3e 100644 --- a/logseq_db_worker/test/test_pure_reducer.ml +++ b/logseq_db_worker/test/test_pure_reducer.ml @@ -444,6 +444,129 @@ let test_asset_failure_is_independent () = Alcotest.(check int) "foreign demand rejected" 0 (List.length stale.effects) ;; +let asset_instructions effects = + List.filter_map + (function + | Core.Run_asset (_, instruction) -> Some instruction + | _ -> None) + effects +;; + +let test_asset_manual_retry_policy () = + let state, scope = worker_open_graph () in + let demanded = Core.step state (asset_demand scope) in + let local = lookup demanded.effects in + let missing = + Core.step + demanded.next + (Core.Asset_completed (scope, Transfer.Cache_checked (local, Ok None))) + in + let download = + List.find_map + (function + | Transfer.Fetch ticket -> Some ticket + | _ -> None) + (asset_instructions missing.effects) + |> Option.get + in + let failed = + Core.step + missing.next + (Core.Asset_completed (scope, Transfer.Downloaded (download, Error Network))) + in + Alcotest.(check bool) + "production never schedules automatic retry" + false + (List.exists + (function + | Transfer.Retry_after _ | Fetch _ -> true + | _ -> false) + (asset_instructions failed.effects)); + let retry = + Core.step + failed.next + (Core.Asset_requested + { graph_generation = scope.graph_generation; event = Retry scope.graph_id }) + in + let local = lookup retry.effects in + Alcotest.(check bool) + "manual retry does not bypass local lookup" + false + (List.exists + (function + | Transfer.Fetch _ -> true + | _ -> false) + (asset_instructions retry.effects)); + let hit = + Core.step + retry.next + (Core.Asset_completed (scope, Transfer.Cache_checked (local, Ok (Some "recovered")))) + in + Alcotest.(check bool) + "local recovery is Ready with no download" + true + (List.exists + (function + | Core.Publish + (Asset_notice (_, Asset_availability { availability = Ready "recovered"; _ })) + -> true + | _ -> false) + hit.effects + && not + (List.exists + (function + | Transfer.Fetch _ -> true + | _ -> false) + (asset_instructions hit.effects))) +;; + +let test_asset_manual_retry_is_unlimited () = + let state, scope = worker_open_graph () in + let demanded = Core.step state (asset_demand scope) in + let rec attempt state local remaining = + let missing = + Core.step + state + (Core.Asset_completed (scope, Transfer.Cache_checked (local, Ok None))) + in + let downloads = + List.filter_map + (function + | Transfer.Fetch ticket -> Some ticket + | _ -> None) + (asset_instructions missing.effects) + in + Alcotest.(check int) + "each explicit local miss starts exactly one download" + 1 + (List.length downloads); + let failed = + Core.step + missing.next + (Core.Asset_completed + (scope, Transfer.Downloaded (List.hd downloads, Error Not_found))) + in + Alcotest.(check bool) + "failed manual attempt has no automatic timer" + false + (List.exists + (function + | Transfer.Retry_after _ -> true + | _ -> false) + (asset_instructions failed.effects)); + if remaining > 0 + then ( + let retry = + Core.step + failed.next + (Core.Asset_requested + { graph_generation = scope.graph_generation; event = Retry scope.graph_id }) + in + attempt retry.next (lookup retry.effects) (remaining - 1)) + in + attempt demanded.next (lookup demanded.effects) 4 +;; + let test_asset_execution_completion_is_delivered () = let state, scope = worker_open_graph () in let request : Sync.asset_request = @@ -874,6 +997,14 @@ let () = "independent failure" `Quick test_asset_failure_is_independent + ; Alcotest.test_case + "manual local-first without automatic retry" + `Quick + test_asset_manual_retry_policy + ; Alcotest.test_case + "manual retries have no attempt quota" + `Quick + test_asset_manual_retry_is_unlimited ] ) ; ( "managed lifecycle" , [ Alcotest.test_case diff --git a/logseq_journal.opam b/logseq_journal.opam index 7c24790..ecc184a 100644 --- a/logseq_journal.opam +++ b/logseq_journal.opam @@ -46,10 +46,10 @@ depends: [ ] pin-depends: [ ["rrbvec.dev" "git+https://github.com/logseq/rrbvec.git#main"] - ["lui.0.1.0" "git+https://github.com/logseq/lui.git#adbdf63fe940157824f29262095bb194ebd21404"] + ["lui.0.1.0" "git+https://github.com/logseq/lui.git#91aecb52a1cba2faaf23aac1d64a0bd1cb6549e7"] ["ocaml-signal.0.1.0" "git+https://github.com/logseq/ocaml-signal.git#main"] - ["datascript_ocaml.dev" "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d"] - ["datascript-ocaml-native.dev" "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d"] + ["datascript_ocaml.dev" "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb"] + ["datascript-ocaml-native.dev" "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb"] ["persistent_sorted_set_ocaml.dev" "git+https://github.com/logseq/persistent-sorted-set-ocaml.git#main"] ["melange-edn-core.0.5.0" "git+https://github.com/logseq/melange-edn.git#main"] ["melange-edn-native.0.5.0" "git+https://github.com/logseq/melange-edn.git#main"] diff --git a/logseq_journal.opam.locked b/logseq_journal.opam.locked index 255ea4d..38ef7a3 100644 --- a/logseq_journal.opam.locked +++ b/logseq_journal.opam.locked @@ -15,15 +15,15 @@ pin-depends: [ ] [ "datascript-ocaml-native.dev" - "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d" + "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb" ] [ "datascript_ocaml.dev" - "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d" + "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb" ] [ "lui.0.1.0" - "git+https://github.com/logseq/lui.git#adbdf63fe940157824f29262095bb194ebd21404" + "git+https://github.com/logseq/lui.git#91aecb52a1cba2faaf23aac1d64a0bd1cb6549e7" ] [ "melange-edn-core.0.5.0" diff --git a/logseq_overlay_db.opam b/logseq_overlay_db.opam index a98799e..7ad76cf 100644 --- a/logseq_overlay_db.opam +++ b/logseq_overlay_db.opam @@ -27,8 +27,8 @@ depends: [ ] pin-depends: [ ["rrbvec.dev" "git+https://github.com/logseq/rrbvec.git#main"] - ["datascript_ocaml.dev" "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d"] - ["datascript-ocaml-native.dev" "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d"] + ["datascript_ocaml.dev" "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb"] + ["datascript-ocaml-native.dev" "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb"] ["melange-edn-core.0.5.0" "git+https://github.com/logseq/melange-edn.git#main"] ["melange-edn-native.0.5.0" "git+https://github.com/logseq/melange-edn.git#main"] ["melange-transit-core.0.1.2" "git+https://github.com/logseq/melange-transit.git#main"] diff --git a/logseq_overlay_db.opam.locked b/logseq_overlay_db.opam.locked index 5bc13dc..5395fb7 100644 --- a/logseq_overlay_db.opam.locked +++ b/logseq_overlay_db.opam.locked @@ -11,11 +11,11 @@ build: ["dune" "build" "-p" name "-j" jobs] pin-depends: [ [ "datascript-ocaml-native.dev" - "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d" + "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb" ] [ "datascript_ocaml.dev" - "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d" + "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb" ] [ "melange-edn-core.0.5.0" diff --git a/logseq_overlay_db/lib/database.ml b/logseq_overlay_db/lib/database.ml index 15056c8..4c41d08 100644 --- a/logseq_overlay_db/lib/database.ml +++ b/logseq_overlay_db/lib/database.ml @@ -8070,6 +8070,193 @@ let journal_membership = function | None | Some _ -> None ;; +let property_holder_dependents before after transaction_data ~maximum = + let definition database entity = + Option.bind (entity_of_ident database "logseq.class/Property") (fun property_class -> + Option.bind + (Option.bind (one database entity "db/ident") ident_of_value) + (fun ident -> property_definition_for_entity database property_class ident entity)) + in + let attributes = Hashtbl.create 16 in + let entities = Hashtbl.create 16 in + List.iter + (List.iter (fun (datom : Datascript.datom) -> Hashtbl.replace entities datom.e ())) + transaction_data; + Hashtbl.iter + (fun entity () -> + let old = definition before entity in + let current = definition after entity in + if old <> current + then + List.iter + (fun (value : Graph.property_summary) -> + Hashtbl.replace attributes value.ident ()) + (Option.to_list old @ Option.to_list current)) + entities; + let holders = Hashtbl.create 16 in + let remaining = ref maximum in + let overflow = ref false in + let inspect database attribute = + let rec collect sequence = + match sequence () with + | Seq.Nil -> () + | Seq.Cons (_, _) when !remaining = 0 -> overflow := true + | Seq.Cons ((datom : Datascript.datom), rest) -> + (* Bound raw work as well as the emitted UUIDs, including malformed holders. *) + decr remaining; + Option.iter + (fun uuid -> Hashtbl.replace holders (Graph.Uuid.to_string uuid) uuid) + (uuid_of_entity database datom.e); + collect rest + in + if not !overflow + then collect (Datascript.datoms database Datascript.Aevt ~a:attribute ()) + in + Hashtbl.iter + (fun attribute () -> + inspect before attribute; + inspect after attribute) + attributes; + Hashtbl.to_seq_values holders |> List.of_seq, !overflow +;; + +let favorite_membership_dependents before after transaction_data ~maximum = + let targets = Hashtbl.create 16 in + List.iter + (List.iter (fun (datom : Datascript.datom) -> Hashtbl.replace targets datom.e ())) + transaction_data; + let memberships = Hashtbl.create 16 in + let remaining = ref maximum in + let overflow = ref false in + let inspect database target = + let rec collect sequence = + match sequence () with + | Seq.Nil -> () + | Seq.Cons (_, _) when !remaining = 0 -> overflow := true + | Seq.Cons ((datom : Datascript.datom), rest) -> + decr remaining; + Option.iter + (fun uuid -> Hashtbl.replace memberships (Graph.Uuid.to_string uuid) uuid) + (uuid_of_entity database datom.e); + collect rest + in + if not !overflow + then + collect + (Datascript.datoms + database + Datascript.Avet + ~a:"block/link" + ~v:(Datascript.Ref target) + ()) + in + Hashtbl.iter + (fun target () -> + inspect before target; + inspect after target) + targets; + Hashtbl.to_seq_values memberships |> List.of_seq, !overflow +;; + +let asset_reference_dependents + before + after + authoritative_before + authoritative_after + transaction_data + ~maximum + = + let changed = Hashtbl.create 16 in + List.iter + (List.iter (fun (datom : Datascript.datom) -> Hashtbl.replace changed datom.e ())) + transaction_data; + let holders = Hashtbl.create 16 in + let remaining = ref maximum in + let overflow = ref false in + let spend () = + if !remaining = 0 + then ( + overflow := true; + false) + else ( + decr remaining; + true) + in + let rec classify reversed sequence = + match sequence () with + | Seq.Nil -> List.rev reversed + | Seq.Cons (_, _) when not (spend () && spend ()) -> List.rev reversed + | Seq.Cons (entity, rest) -> + (* Charge both eligibility point reads before touching a candidate. The + transaction input is finite; dependent database work shares one budget. *) + let changed = + match + ( uuid_of_entity authoritative_after entity + , uuid_of_entity authoritative_before entity ) + with + | None, None -> false + | current, old -> + let uuid = + Option.get + (match current with + | Some _ -> current + | None -> old) + in + (match asset_descriptor_at before uuid, asset_descriptor_at after uuid with + | Ok old, Ok current -> Option.is_some old <> Option.is_some current + | _ -> true) + in + classify (if changed then entity :: reversed else reversed) rest + in + let targets = classify [] (Hashtbl.to_seq_keys changed) in + let inspect database = + let property_class = entity_of_ident database "logseq.class/Property" in + let admitted attribute = + attribute = "block/refs" + || Option.fold + ~none:false + ~some:(fun property_class -> + Option.is_some + (property_definition_with_class database property_class attribute)) + property_class + in + let rec collect sequence = + match sequence () with + | Seq.Nil -> () + | Seq.Cons (_, _) when not (spend ()) -> () + | Seq.Cons ((datom : Datascript.datom), rest) -> + Option.iter + (fun uuid -> Hashtbl.replace holders (Graph.Uuid.to_string uuid) uuid) + (uuid_of_entity database datom.e); + collect rest + in + let rec attributes = function + | [] -> () + | _ when !overflow -> () + | (attribute, (schema : Datascript.schema_attr)) :: rest -> + if spend () && schema.value_type = Some Datascript.RefType && admitted attribute + then + List.iter + (fun target -> + if (not !overflow) && spend () + then + collect + (Datascript.datoms + database + Datascript.Avet + ~a:attribute + ~v:(Datascript.Ref target) + ())) + targets; + attributes rest + in + if targets <> [] && not !overflow then attributes (Datascript.schema database) + in + inspect authoritative_before; + inspect authoritative_after; + Hashtbl.to_seq_values holders |> List.of_seq, !overflow +;; + let logical_change_footprint ~before ~after @@ -8081,6 +8268,23 @@ let logical_change_footprint let direct = changed_entity_uuids authoritative_before authoritative_after transaction_data in + let property_holders, property_overflow = + property_holder_dependents + authoritative_before + authoritative_after + transaction_data + ~maximum:before.owner.dependencies.limits.change_max_items + in + let favorite_memberships, favorite_overflow = + favorite_membership_dependents authoritative_before authoritative_after transaction_data + ~maximum:before.owner.dependencies.limits.change_max_items + in + let asset_holders, asset_overflow = + asset_reference_dependents before after authoritative_before authoritative_after transaction_data + ~maximum:before.owner.dependencies.limits.change_max_items + in + let asset_pages, asset_blocks = List.partition (fun uuid -> + Option.is_some (logical_page_at before uuid) || Option.is_some (logical_page_at after uuid)) asset_holders in let outbox_blocks, outbox_pages = List.fold_left (fun (blocks, pages) (footprint : effect_footprint) -> @@ -8088,7 +8292,9 @@ let logical_change_footprint ([], []) outbox_effects in - let page_candidates = List.sort_uniq Graph.Uuid.compare (direct @ outbox_pages) in + let page_candidates = + List.sort_uniq Graph.Uuid.compare (direct @ outbox_pages @ property_holders) + in let changed_pages = List.filter (fun uuid -> logical_page_at before uuid <> logical_page_at after uuid) @@ -8122,7 +8328,9 @@ let logical_change_footprint collect maximum_dependents [] changed_pages in let block_candidates = - List.sort_uniq Graph.Uuid.compare (direct @ outbox_blocks @ page_dependent_blocks) + List.sort_uniq + Graph.Uuid.compare + (direct @ outbox_blocks @ page_dependent_blocks @ property_holders) in let block_uuids = List.filter @@ -8131,6 +8339,7 @@ let logical_change_footprint let after_value = logical_block_at after uuid in before_value <> after_value) block_candidates + |> fun changed -> List.sort_uniq Graph.Uuid.compare (changed @ favorite_memberships @ asset_blocks) in let parents, membership_pages = List.fold_left @@ -8173,9 +8382,9 @@ let logical_change_footprint @ if journal_changed then [ Types.Journal_index_interest ] else [] in ( block_uuids - , List.sort_uniq Graph.Uuid.compare (changed_pages @ membership_pages) + , List.sort_uniq Graph.Uuid.compare (changed_pages @ membership_pages @ asset_pages) , List.sort_uniq compare structure_interests - , dependent_overflow ) + , dependent_overflow || property_overflow || favorite_overflow || asset_overflow ) ;; let storage_session_error = function @@ -8736,7 +8945,7 @@ let commit_authoritative_candidate database prepared = transitioned_records) in let changed = - block_uuids <> [] || page_uuids <> [] || structure_interests <> [] + fanout_overflow || block_uuids <> [] || page_uuids <> [] || structure_interests <> [] in database.projection <- (if changed then candidate_projection else before_projection); @@ -8940,7 +9149,10 @@ let get_assets_under_roots snapshot ~recursive ~roots ~limit ~cursor = else ( match logical_tree_candidate snapshot tree_cache uuid with | Some candidate -> live (uuid :: seen) candidate.tree_parent - | None -> Option.is_some entity)) + | None -> + (match logical_page_at ~cache:tree_cache.hydration snapshot uuid with + | Some { page; _ } -> not page.recycled + | None -> Option.is_some entity))) in let peek sequence = match sequence () with diff --git a/logseq_overlay_db/test/test_asset_reads.ml b/logseq_overlay_db/test/test_asset_reads.ml index b2dfd8e..011aa4e 100644 --- a/logseq_overlay_db/test/test_asset_reads.ml +++ b/logseq_overlay_db/test/test_asset_reads.ml @@ -329,6 +329,188 @@ let local_asset_import database = | _ -> Alcotest.fail "uploaded metadata missing") ;; +let local_page_asset_import database = + let page = id 95 + and text = id 96 + and asset = id 97 in + let expected = + with_snapshot database (fun snapshot -> + match D.get_pages snapshot [ page ] |> get with + | [ Types.Missing_page { revision; _ } ] -> + D.write_precondition ~blocks:[] ~pages:[ page, revision ] ~scopes:[] |> get + | _ -> Alcotest.fail "local journal page unexpectedly exists") + in + ignore + (D.commit_local + database + ~expected + (Types.Create_journal_page + { mutation_id = T.mutation_uuid 1200 + ; page + ; title = "2026-10-09" + ; journal_day = 20261009 + }) + |> get); + let expected = + T.insert_precondition database ~parent:page ~behavior:"local capture text" + in + ignore + (D.commit_local + database + ~expected + (Types.Insert_blocks + { mutation_id = T.mutation_uuid 1201 + ; parent = page + ; tree = { uuid = text; title = "Local capture"; children = [] } + ; asset = None + }) + |> get); + let expected = + with_snapshot database (fun snapshot -> + let revision = + match D.get_blocks snapshot [ text ] |> get with + | [ Types.Present_block block ] -> block.revision + | _ -> Alcotest.fail "local capture text missing" + in + match + D.get_structure + snapshot + (Types.Children { parent = text; limit = 1; cursor = None }) + |> get + with + | Types.Children_result { revision_scope; scope_revision; _ } -> + D.write_precondition + ~blocks:[ text, revision ] + ~pages:[] + ~scopes:[ revision_scope, scope_revision ] + |> get + | _ -> Alcotest.fail "local capture children missing") + in + let version = A.version ~checksum:(String.make 64 'e') ~file_type:"png" |> get in + let mutation = + Types.Insert_blocks + { mutation_id = T.mutation_uuid 1202 + ; parent = text + ; tree = { uuid = asset; title = "Pending local PNG"; children = [] } + ; asset = Some { replace_reference = None; version; size = 11L } + } + in + ignore (D.commit_local database ~expected mutation |> get); + T.require + (Option.is_some (D.inspect_local_mutation database mutation |> get)) + "local capture asset has no durable receipt"; + with_snapshot database (fun snapshot -> + let descriptors = D.get_asset_descriptors snapshot [ asset ] |> get in + (match descriptors with + | [ { A.uuid + ; source = Managed None + ; current_checksum = Some checksum + ; size = Some 11L + ; _ + } + ] -> + T.require + (G.Uuid.equal uuid asset && checksum = version.checksum) + "local pending descriptor differs from durable import" + | _ -> Alcotest.fail "local pending Managed None descriptor missing"); + let direct = + D.get_assets_under_roots + snapshot + ~recursive:false + ~roots:[ asset ] + ~limit:1 + ~cursor:None + |> get + in + T.require + (direct.assets = descriptors) + "local asset root lost its logical page ancestry"; + T.require + (collect snapshot [ page ] = [ asset ]) + "local page subtree omitted pending asset"; + T.require + (collect snapshot [ text ] = [ asset ]) + "local text subtree omitted pending asset"; + T.require + (collect ~recursive:false snapshot [ page; text ] = []) + "direct page/text query included child assets"; + T.require (collect snapshot [ id 99999 ] = []) "missing logical root became live"; + let expected = + T.delete_precondition database ~block:text ~behavior:"delete local capture" + in + ignore + (D.commit_local + database + ~expected + (Types.Delete_blocks { mutation_id = T.mutation_uuid 1203; root = text }) + |> get); + with_snapshot database (fun current -> + T.require + (collect current [ page; text; asset ] = []) + "tombstoned local ancestor retained a pending asset"); + T.require + (collect snapshot [ page ] = [ asset ]) + "local asset deletion changed pinned snapshot") +;; + +let recycled_asset_ancestry database = + seed database; + apply database 2 [ add 0 "logseq.property/deleted-at" (Int64 1_788_192_000_001L) ]; + with_snapshot database (fun snapshot -> + (match D.get_pages snapshot [ id 0 ] |> get with + | [ Types.Present_page { value; _ } ] -> + T.require value.page.recycled "fixture page is not recycled" + | _ -> Alcotest.fail "recycled page missing"); + T.require (collect snapshot [ id 0; id 1 ] = []) "recycled ancestry became live"; + T.require + (collect ~recursive:false snapshot [ id 3 ] = [ id 3 ]) + "standalone authoritative asset was lost") +;; + +let cyclic_asset_ancestry database = + seed database; + apply database 2 [ add 1 "block/parent" (Int 40002) ]; + with_snapshot database (fun snapshot -> + match + D.get_assets_under_roots + snapshot + ~recursive:false + ~roots:[ id 1 ] + ~limit:1 + ~cursor:None + with + | Error (Types.Fatal_read_state _) -> () + | _ -> Alcotest.fail "cyclic ancestry did not fail explicitly") +;; + +let bounded_asset_ancestry database = + seed database; + apply + database + 2 + (List.init 257 (fun offset -> + let n = 1000 + offset + and parent = if offset = 0 then 0 else 999 + offset in + [ add n "block/uuid" (Uuid (G.Uuid.to_string (id n))) + ; add n "block/title" (String "Deep asset ancestor") + ; add n "block/parent" (Int (40000 + parent)) + ; add n "block/page" (Int 40000) + ; add n "block/order" (String "a") + ]) + |> List.concat); + with_snapshot database (fun snapshot -> + match + D.get_assets_under_roots + snapshot + ~recursive:false + ~roots:[ id 1256 ] + ~limit:1 + ~cursor:None + with + | Error Types.Read_limit_exceeded -> () + | _ -> Alcotest.fail "asset ancestry exceeded its explicit depth bound") +;; + let reuse_asset_reference database = seed database; let expected block = @@ -616,6 +798,333 @@ let restore_asset_mutations () = | _ -> Alcotest.fail "asset mutations did not survive reopen"))))) ;; +let property_holder_changes database register = + seed database; + let ident = "asset-test.property/reference" in + let definition = + [ add 8 "block/uuid" (Uuid (G.Uuid.to_string (id 8))) + ; add 8 "block/title" (String "Asset reference") + ; add 8 "block/tags" (Array [ Keyword "db/ident"; Keyword "logseq.class/Property" ]) + ; add 8 "logseq.property/type" (Keyword "node") + ] + in + apply + database + 2 + ([ add 8 "db/ident" (Keyword ident) + ; add 8 "db/valueType" (Keyword "db.type/ref") + ; add 8 "db/cardinality" (Keyword "db.cardinality/one") + ; add 1 ident (Int 40006) + ; add 9 ident (Int 40006) + ] + @ if register then [] else definition); + let subscriber, predecessor = D.listen database |> get in + D.release_snapshot predecessor; + let changes = ref [] in + D.activate_subscription subscriber ~notify:(fun change -> changes := change :: !changes) + |> get; + apply + database + 3 + (if register + then definition + else + [ Transit.Array + [ Keyword "db/retract" + ; Int 40008 + ; Keyword "block/tags" + ; Array [ Keyword "db/ident"; Keyword "logseq.class/Property" ] + ] + ]); + D.unlisten subscriber; + with_snapshot database (fun snapshot -> + T.require + (List.mem (id 6) (collect ~recursive:false snapshot [ id 1 ]) = register) + "property membership did not change"; + T.require + (List.mem (id 6) (collect ~recursive:false snapshot [ id 9 ]) = register) + "page property membership did not change"); + match !changes with + | [ Types.Exact { block_uuids; page_uuids; _ } ] -> + T.require (List.mem (id 1) block_uuids) "property definition omitted block holder"; + T.require (List.mem (id 9) page_uuids) "property definition omitted page holder" + | _ -> Alcotest.fail "bounded property definition change must publish one exact event" +;; + +let property_holder_overflow () = + let behavior = "bounded property holder fanout" in + let limits = { (T.limits ~behavior) with change_max_items = 2 } in + T.with_database_using_limits ~behavior limits (fun database -> + seed database; + let ident = "asset-test.property/many" in + apply + database + 2 + [ add 8 "db/ident" (Keyword ident) + ; add 8 "db/valueType" (Keyword "db.type/ref") + ; add 8 "db/cardinality" (Keyword "db.cardinality/one") + ; add 1 ident (Int 40005) + ; add 2 ident (Int 40005) + ; add 9 ident (Int 40005) + ]; + let subscriber, predecessor = D.listen database |> get in + let revision = (D.snapshot_version predecessor).projection_revision in + D.release_snapshot predecessor; + let changes = ref [] in + D.activate_subscription subscriber ~notify:(fun change -> + changes := change :: !changes) + |> get; + apply + database + 3 + [ add 8 "block/uuid" (Uuid (G.Uuid.to_string (id 8))) + ; add 8 "block/title" (String "Many asset references") + ; add 8 "block/tags" (Array [ Keyword "db/ident"; Keyword "logseq.class/Property" ]) + ; add 8 "logseq.property/type" (Keyword "node") + ]; + D.unlisten subscriber; + T.require + (match !changes with + | [ Types.Projection_resync_required _ ] -> true + | _ -> false) + "property fanout overflow did not publish resync"; + with_snapshot database (fun snapshot -> + T.require + (not + (Types.Projection_revision.equal + revision + (D.snapshot_version snapshot).projection_revision)) + "property fanout resync did not advance projection revision")) +;; + +let asset_target_classification_budget () = + let behavior = "bounded eligibility classification" in + let limits = { (T.limits ~behavior) with change_max_items = 2 } in + T.with_database_using_limits ~behavior limits (fun database -> + apply + database + 1 + (List.init 8 (fun n -> + add (100 + n) "block/uuid" (Uuid (G.Uuid.to_string (id (100 + n)))))); + with_snapshot database (fun snapshot -> + T.require + (List.for_all + (function + | Types.Missing_block _ -> true + | _ -> false) + (D.get_blocks snapshot (List.init 8 (fun n -> id (100 + n))) |> get)) + "unresolved reference stubs must not be projected blocks"); + let subscriber, predecessor = D.listen database |> get in + let revision = (D.snapshot_version predecessor).projection_revision in + D.release_snapshot predecessor; + let changes = ref [] in + D.activate_subscription subscriber ~notify:(fun change -> + changes := change :: !changes) + |> get; + apply + database + 2 + (List.init 8 (fun n -> add (100 + n) "block/title" (String "Unresolved reference"))); + D.unlisten subscriber; + T.require + (match !changes with + | [ Types.Projection_resync_required _ ] -> true + | _ -> false) + "eligibility classification exhausted its budget without explicit resync"; + with_snapshot database (fun snapshot -> + T.require + (not + (Types.Projection_revision.equal + revision + (D.snapshot_version snapshot).projection_revision)) + "classification resync must advance projection revision")) +;; + +let negative_asset_holder_changes + ?(property_type = "asset") + ?(holder_page = false) + ?(expect_resync = false) + typed + database + = + seed database; + let attribute = if typed then "asset-test.property/negative" else "block/refs" in + let definition = + if typed + then + [ add 8 "db/ident" (Keyword attribute) + ; add 8 "db/valueType" (Keyword "db.type/ref") + ; add 8 "db/cardinality" (Keyword "db.cardinality/one") + ; add 8 "block/uuid" (Uuid (G.Uuid.to_string (id 8))) + ; add 8 "block/title" (String "Reference") + ; add 8 "block/tags" (Array [ Keyword "db/ident"; Keyword "logseq.class/Property" ]) + ; add 8 "logseq.property/type" (Keyword property_type) + ] + else [] + in + apply + database + 2 + (definition + @ [ add 90 "block/uuid" (Uuid (G.Uuid.to_string (id 90))) + ; add 90 "block/name" (String "negative-assets") + ; add 90 "block/title" (String "Page") + ; add 91 "block/uuid" (Uuid (G.Uuid.to_string (id 91))) + ; add 91 "block/title" (String "Hidden holder") + ; add 91 "block/parent" (Int 40090) + ; add 91 "block/page" (Int 40090) + ; add 91 "block/order" (String "a") + ; add 92 "block/uuid" (Uuid (G.Uuid.to_string (id 92))) + ; add 92 "block/title" (String "Initially not an asset") + ; add + (if holder_page then 90 else 91) + attribute + (Array [ Keyword "block/uuid"; Uuid (G.Uuid.to_string (id 92)) ]) + ]); + with_snapshot database (fun snapshot -> + T.require + (collect snapshot [ id 90 ] = []) + "negative page must initially have no assets"); + let subscription, predecessor = D.listen database |> get in + D.release_snapshot predecessor; + let changes = ref [] in + D.activate_subscription subscription ~notify:(fun change -> + changes := change :: !changes) + |> get; + apply + database + 3 + [ add 92 "block/tags" (Array [ Keyword "db/ident"; Keyword "logseq.class/Asset" ]) + ; add 92 "logseq.property.asset/type" (String "png") + ]; + D.unlisten subscription; + with_snapshot database (fun snapshot -> + T.require + (collect snapshot [ id 90 ] = [ id 92 ]) + "new asset must become discoverable"); + T.require + (List.exists + (function + | Types.Projection_resync_required _ when expect_resync -> true + | Types.Exact { block_uuids; page_uuids; _ } when not expect_resync -> + if holder_page + then List.mem (id 90) page_uuids + else List.mem (id 91) block_uuids + | _ -> false) + !changes) + "asset eligibility change omitted unchanged hidden reference holder" +;; + +let negative_favorite_target_changes ?(link_changed = false) database = + seed database; + let favorite_page = + with_snapshot database (fun snapshot -> + Option.value + (D.get_favorites snapshot ~limit:50 ~cursor:None |> get).favorites_page + ~default:(id 99)) + in + let page_fields = + if G.Uuid.equal favorite_page (id 99) + then + [ add 99 "block/uuid" (Uuid (G.Uuid.to_string (id 99))) + ; add 99 "block/name" (String "$$$favorites") + ; add 99 "block/title" (String "Favorites") + ] + else [] + in + let favorite_page_ref = + Transit.Array [ Keyword "block/uuid"; Uuid (G.Uuid.to_string favorite_page) ] + in + apply + database + 2 + (page_fields + @ [ add 98 "block/uuid" (Uuid (G.Uuid.to_string (id 98))) + ; add 98 "block/title" (String "") + ; add 98 "block/parent" favorite_page_ref + ; add 98 "block/page" favorite_page_ref + ; add 98 "block/order" (String "a0") + ; add 96 "block/uuid" (Uuid (G.Uuid.to_string (id 96))) + ] + @ + if link_changed + then + [ add 96 "block/title" (String "New favorite") + ; add 96 "block/parent" (Int 40000) + ; add 96 "block/page" (Int 40000) + ; add 96 "block/order" (String "a1") + ] + else + [ add + 98 + "block/link" + (Array [ Keyword "block/uuid"; Uuid (G.Uuid.to_string (id 96)) ]) + ]); + with_snapshot database (fun snapshot -> + T.require + ((D.get_favorites snapshot ~limit:50 ~cursor:None |> get).items = []) + "incomplete favorite target must initially be absent"); + let subscription, predecessor = D.listen database |> get in + D.release_snapshot predecessor; + let changes = ref [] in + D.activate_subscription subscription ~notify:(fun change -> + changes := change :: !changes) + |> get; + apply + database + 3 + (if link_changed + then + [ add + 98 + "block/link" + (Array [ Keyword "block/uuid"; Uuid (G.Uuid.to_string (id 96)) ]) + ] + else + [ add 96 "block/title" (String "New favorite") + ; add 96 "block/parent" (Int 40000) + ; add 96 "block/page" (Int 40000) + ; add 96 "block/order" (String "a1") + ]); + D.unlisten subscription; + with_snapshot database (fun snapshot -> + let favorites = D.get_favorites snapshot ~limit:50 ~cursor:None |> get in + if List.length favorites.items <> 1 + then ( + let target = + match D.get_blocks snapshot [ id 96 ] |> get with + | [ Types.Present_block { value; _ } ] -> + Printf.sprintf + "%s parent=%s page=%s" + value.block.title + (G.Uuid.to_string value.block.parent) + (G.Uuid.to_string value.block.page) + | _ -> "missing" + in + let children = + match + D.get_structure + snapshot + (Types.Children { parent = favorite_page; limit = 200; cursor = None }) + |> get + with + | Children_result { items; _ } -> List.length items + | _ -> -1 + in + Alcotest.failf + "completed target not favorite: target=%s members=%d favoritesPage=%s" + target + children + (Option.fold ~none:"none" ~some:G.Uuid.to_string favorites.favorites_page))); + T.require + (List.exists + (function + | Types.Exact { block_uuids; _ } -> List.mem (id 98) block_uuids + | _ -> false) + !changes) + "target eligibility change omitted its unchanged favorite membership UUID" +;; + let () = Alcotest.run "asset reads" @@ -624,6 +1133,48 @@ let () = ; Alcotest.test_case "reopen references" `Quick restore_asset_references ; Alcotest.test_case "reopen replacement" `Quick restore_binary_replacement ] ) + ; ( "property membership changes" + , [ T.database_case "register existing reference holders" (fun db -> + property_holder_changes db true) + ; T.database_case "unregister existing reference holders" (fun db -> + property_holder_changes db false) + ; Alcotest.test_case + "bounded holder fanout advances revision" + `Quick + property_holder_overflow + ; T.database_case + "negative favorite target becomes valid" + (negative_favorite_target_changes ~link_changed:false) + ; T.database_case + "negative favorite membership gains link" + (negative_favorite_target_changes ~link_changed:true) + ; T.database_case + "negative refs holder becomes discoverable" + (negative_asset_holder_changes false) + ; T.database_case + "negative typed asset holder becomes discoverable" + (negative_asset_holder_changes true) + ; Alcotest.test_case + "asset eligibility classification is bounded" + `Quick + asset_target_classification_budget + ; Alcotest.test_case + "bounded negative asset fanout requires resync" + `Quick + (fun () -> + let behavior = "bounded negative reference fanout" in + let limits = { (T.limits ~behavior) with change_max_items = 2 } in + T.with_database_using_limits + ~behavior + limits + (negative_asset_holder_changes ~expect_resync:true false)) + ; T.database_case + "negative typed node holder becomes discoverable" + (negative_asset_holder_changes ~property_type:"node" true) + ; T.database_case + "negative typed page holder becomes discoverable" + (negative_asset_holder_changes ~holder_page:true true) + ] ) ; ( "bounded graph metadata" , List.map (fun (n, f) -> T.database_case n f) @@ -634,6 +1185,10 @@ let () = ; "reuse asset reference", reuse_asset_reference ; "direct visible assets", direct_assets ; "local asset import", local_asset_import + ; "local journal capture asset ancestry", local_page_asset_import + ; "recycled asset ancestry", recycled_asset_ancestry + ; "cyclic asset ancestry", cyclic_asset_ancestry + ; "bounded asset ancestry", bounded_asset_ancestry ; "direct and subtree discovery", discovery ; "cursor fences", stale ; "effective overlay", deletion diff --git a/logseq_sync.opam b/logseq_sync.opam index 2220e66..1d45d19 100644 --- a/logseq_sync.opam +++ b/logseq_sync.opam @@ -34,8 +34,8 @@ depends: [ ] depexts: ["zlib"] pin-depends: [ - ["datascript_ocaml.dev" "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d"] - ["datascript-ocaml-native.dev" "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d"] + ["datascript_ocaml.dev" "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb"] + ["datascript-ocaml-native.dev" "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb"] ["melange-edn-core.0.5.0" "git+https://github.com/logseq/melange-edn.git#main"] ["melange-edn-native.0.5.0" "git+https://github.com/logseq/melange-edn.git#main"] ["melange-transit-core.0.1.2" "git+https://github.com/logseq/melange-transit.git#main"] diff --git a/logseq_sync.opam.locked b/logseq_sync.opam.locked index fe315e9..e8f8f3f 100644 --- a/logseq_sync.opam.locked +++ b/logseq_sync.opam.locked @@ -140,11 +140,11 @@ pin-depends: [ ] [ "datascript-ocaml-native.dev" - "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d" + "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb" ] [ "datascript_ocaml.dev" - "git+https://github.com/logseq/datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d" + "git+https://github.com/logseq/datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb" ] [ "melange-edn-core.0.5.0" diff --git a/logseq_sync/lib/pure_reducer/asset_transfer.ml b/logseq_sync/lib/pure_reducer/asset_transfer.ml index 121b90c..d506e62 100644 --- a/logseq_sync/lib/pure_reducer/asset_transfer.ml +++ b/logseq_sync/lib/pure_reducer/asset_transfer.ml @@ -263,6 +263,7 @@ let same_version a b = let update_descriptor t descriptor = match Assets.find_opt descriptor.Asset.uuid t.entries with | None -> t, [] + | Some { phase = Cached _; _ } -> t, [] | Some entry when same_version entry.descriptor descriptor -> ( { t with entries = Assets.add descriptor.uuid { entry with descriptor } t.entries } , [] ) @@ -410,14 +411,17 @@ let raw_step t = function | Downloaded (ticket, result) -> completed t ticket (Result.map Option.some result) false | Retry uuid -> - ( { t with - entries = - Assets.update - uuid - (Option.map (fun entry -> { (resume_entry entry) with attempts = 0 })) - t.entries - } - , [] ) + (match Assets.find_opt uuid t.entries with + | Some ({ phase = Rejected _; _ } as entry) -> + ( { t with + entries = + Assets.add + uuid + { entry with phase = Pending; checked = false; attempts = 0 } + t.entries + } + , [] ) + | None | Some _ -> t, []) | Retry_elapsed id -> ( { t with entries = diff --git a/logseq_sync/test/test_asset_transfer.ml b/logseq_sync/test/test_asset_transfer.ml index f7eee76..db7422b 100644 --- a/logseq_sync/test/test_asset_transfer.ml +++ b/logseq_sync/test/test_asset_transfer.ml @@ -260,6 +260,146 @@ let lock_and_network () = check "online resumes demand" 1 (List.length (fetches e)) ;; +let failed_transfer () = + let t, effects = replace (initial ()) "visible" Foreground [ asset 2 ] in + let t, effects = T.step t (Cache_checked (only (checks effects), Ok None)) in + let download = only (fetches effects) in + let t, effects = T.step t (Downloaded (download, Error Network)) in + t, download, effects +;; + +let manual_retry_cached_file () = + let t, _, _ = failed_transfer () in + let t, effects = T.step t (Retry (uuid 2)) in + check "manual retry checks local before downloading" 1 (List.length (checks effects)); + check + "manual retry does not fetch before cache answer" + 0 + (List.length (fetches effects)); + let t, effects = + T.step t (Cache_checked (only (checks effects), Ok (Some "recovered"))) + in + Alcotest.(check bool) "recovered local file becomes Ready" true (ready t "visible"); + check "local recovery needs no download" 0 (List.length (fetches effects)) +;; + +let manual_retry_missing_file () = + let t, _, _ = failed_transfer () in + let t, effects = T.step t (Retry (uuid 2)) in + check "retry starts one local lookup" 1 (List.length (checks effects)); + let t, effects = T.step t (Cache_checked (only (checks effects), Ok None)) in + check "missing local starts one download" 1 (List.length (fetches effects)); + let t, effects = T.step t (Downloaded (only (fetches effects), Ok "downloaded")) in + Alcotest.(check bool) "download completes Ready" true (ready t "visible"); + check "completion does not start another download" 0 (List.length (fetches effects)) +;; + +let duplicate_retry_while_loading () = + let t, _, _ = failed_transfer () in + let t, effects = T.step t (Retry (uuid 2)) in + check "retry starts local lookup" 1 (List.length (checks effects)); + let local = only (checks effects) in + let t, effects = T.step t (Retry (uuid 2)) in + check "retry while checking has no effects" 0 (List.length effects); + Alcotest.(check bool) "local lookup stays current" true (T.ticket_current t local); + let t, effects = T.step t (Cache_checked (local, Ok None)) in + let download = only (fetches effects) in + let t, effects = T.step t (Retry (uuid 2)) in + check "retry while fetching has no effects" 0 (List.length effects); + Alcotest.(check bool) "download stays current" true (T.ticket_current t download) +;; + +let ready_does_not_refresh () = + let t, effects = replace (initial ()) "visible" Foreground [ asset 2 ] in + let t, _ = T.step t (Cache_checked (only (checks effects), Ok (Some "displayed"))) in + let t, effects = T.step t (Retry (uuid 2)) in + check "Ready ignores manual retry" 0 (List.length effects); + let descriptors = + [ asset 2 + ; asset ~checksum:(String.make 64 'b') 2 + ; asset ~remote:false 2 + ; get + (A.create + ~uuid:(uuid 2) + ~source:(External "https://example/new-image") + ~current_checksum:None + ~size:None + ~dimensions:None) + ] + in + let t = + List.fold_left + (fun t descriptor -> + let t, effects = T.step t (Descriptor_changed descriptor) in + check + "Ready descriptor refresh has no IO or notification" + 0 + (List.length effects); + Alcotest.(check bool) "displayed file remains Ready" true (ready t "visible"); + t) + t + descriptors + in + let _, effects = + replace t "visible" Foreground [ asset ~checksum:(String.make 64 'b') 2 ] + in + check "replacing demand keeps displayed resource" 0 (List.length effects) +;; + +let manual_retry_fences_old_work () = + let t, old, effects = failed_transfer () in + let timer = + List.find_map + (function + | T.Retry_after { id; _ } -> Some id + | _ -> None) + effects + |> Option.get + in + let t, effects = T.step t (Retry (uuid 2)) in + check "retry starts fresh local lookup" 1 (List.length (checks effects)); + let local = only (checks effects) in + let t, effects = T.step t (Retry_elapsed timer) in + check "old automatic timer cannot duplicate manual retry" 0 (List.length effects); + let t, effects = T.step t (Downloaded (old, Ok "stale")) in + Alcotest.(check bool) + "late old download releases its handle" + true + (release_present effects); + Alcotest.(check bool) + "fresh local ticket survives stale completion" + true + (T.ticket_current t local); + let t, effects = T.step t (Release "visible") in + Alcotest.(check bool) + "release cancels local lookup" + true + (List.mem (T.Cancel local) effects); + let t, effects = T.step t (Cache_checked (local, Ok (Some "stale"))) in + Alcotest.(check bool) + "released completion cleans resource" + true + (release_present effects); + check + "released consumer is not resurrected" + 0 + (List.length (T.availability t ~consumer:"visible")); + let scope = { scope with graph_generation = scope.graph_generation + 1 } in + let fresh = + T.create + (get (T.config ~active:2 ~foreground_reserved:1 ~pending:3 ~retries:2)) + ~scope + ~online:true + ~unlocked:true + in + let fresh, effects = T.step fresh (Downloaded (old, Ok "stale")) in + Alcotest.(check bool) + "old scope completion cleans resource" + true + (release_present effects); + check "old scope cannot create demand" 0 (T.pending_count fresh) +;; + let () = Alcotest.run "asset transfer" @@ -278,6 +418,11 @@ let () = ; "duplicate completion", duplicate ; "pending foreground capacity", foreground_queue_capacity ; "lock and network", lock_and_network + ; "manual retry finds local file", manual_retry_cached_file + ; "manual retry downloads only after local miss", manual_retry_missing_file + ; "manual retry is inert while loading", duplicate_retry_while_loading + ; "Ready ignores Retry and descriptor refresh", ready_does_not_refresh + ; "manual retry fences old work", manual_retry_fences_old_work ] ) ] ;; diff --git a/swift/App.swift b/swift/App.swift index 32e0984..bbd063b 100644 --- a/swift/App.swift +++ b/swift/App.swift @@ -78,7 +78,7 @@ private struct JournalHost: View { } .sheet(isPresented: $signInPresented) { NavigationStack { - JournalAuthenticationView(api: JournalAmplifyAuthentication()) { + JournalAuthenticationView(api: JournalCognitoNative.shared) { platform.refreshAuthentication() signInPresented = false } diff --git a/swift/JournalAmplifyAuthentication.swift b/swift/JournalAmplifyAuthentication.swift deleted file mode 100644 index 7870405..0000000 --- a/swift/JournalAmplifyAuthentication.swift +++ /dev/null @@ -1,77 +0,0 @@ -import Amplify -import AWSCognitoAuthPlugin -import Foundation - -@MainActor struct JournalAmplifyAuthentication: JournalAuthenticationAPI { - struct Failure: LocalizedError { - let message: String - var errorDescription: String? { message } - } - - func perform(_ command: JournalAuthenticationCommand) async throws -> JournalAuthenticationStep { - do { - try JournalAmplifySession.configure() - switch command { - case .signIn(let username, let password): - return try Self.step(await Amplify.Auth.signIn(username: username, password: password).nextStep) - case .confirm(let response): - return try Self.step(await Amplify.Auth.confirmSignIn(challengeResponse: response).nextStep) - case .requestReset(let username): - let result = try await Amplify.Auth.resetPassword(for: username) - switch result.nextStep { - case .confirmResetPasswordWithCode(let delivery, _): return .resetConfirmation(Self.destination(delivery)) - case .done: return .signIn - } - case .confirmReset(let username, let code, let password): - try await Amplify.Auth.confirmResetPassword(for: username, with: password, confirmationCode: code) - return .signIn - case .confirmAccount(let username, let code): - let result = try await Amplify.Auth.confirmSignUp(for: username, confirmationCode: code) - switch result.nextStep { - case .done, .completeAutoSignIn: return .signIn - case .confirmUser(let delivery, _, _): return .confirmAccount(delivery.map(Self.destination) ?? "your email or phone") - } - case .resendAccount(let username): - return .confirmAccount(Self.destination(try await Amplify.Auth.resendSignUpCode(for: username))) - } - } catch let error as AuthError { - throw Failure(message: error.errorDescription) - } - } - - static func step(_ step: AuthSignInStep) throws -> JournalAuthenticationStep { - typealias Challenge = JournalAuthenticationStep.Challenge - switch step { - case .done: return .complete - case .resetPassword: return .resetRequest - case .confirmSignUp: return .confirmAccount("your email or phone") - case .confirmSignInWithSMSMFACode(let delivery, _), .confirmSignInWithOTP(let delivery): - return .challenge(Challenge(title: "Verify your sign-in", prompt: "Verification code", input: .oneTimeCode, detail: "Code sent to \(destination(delivery))")) - case .confirmSignInWithTOTPCode: - return .challenge(Challenge(title: "Verify your sign-in", prompt: "Authenticator code", input: .oneTimeCode)) - case .confirmSignInWithNewPassword: - return .challenge(Challenge(title: "Choose a new password", prompt: "New password", input: .newPassword)) - case .confirmSignInWithPassword: - return .challenge(Challenge(title: "Enter your password", prompt: "Password", input: .password)) - case .confirmSignInWithCustomChallenge(let info): - return .challenge(Challenge(title: "Verify your sign-in", prompt: info?["prompt"] ?? "Verification response")) - case .continueSignInWithTOTPSetup(let details): - return .challenge(Challenge(title: "Set up an authenticator", prompt: "Authenticator code", input: .oneTimeCode, - detail: "Add this setup key to your authenticator, then enter its code: \(details.sharedSecret)")) - case .continueSignInWithEmailMFASetup: - return .challenge(Challenge(title: "Set up email verification", prompt: "Email address", input: .emailAddress)) - case .continueSignInWithMFASelection(let choices), .continueSignInWithMFASetupSelection(let choices): - return .challenge(Challenge(title: "Choose a verification method", prompt: "Method", choices: choices.map(\.challengeResponse).sorted())) - case .continueSignInWithFirstFactorSelection(let choices): - return .challenge(Challenge(title: "Choose a sign-in method", prompt: "Method", choices: choices.map(\.challengeResponse).sorted())) - } - } - - static func destination(_ details: AuthCodeDeliveryDetails) -> String { - switch details.destination { - case .email(let value): return value ?? "your email" - case .phone(let value), .sms(let value): return value ?? "your phone" - case .unknown(let value): return value ?? "your account contact" - } - } -} diff --git a/swift/JournalAmplifySession.swift b/swift/JournalAmplifySession.swift deleted file mode 100644 index 2a31d3d..0000000 --- a/swift/JournalAmplifySession.swift +++ /dev/null @@ -1,73 +0,0 @@ -import Amplify -import AWSCognitoAuthPlugin -import Foundation - -@MainActor final class JournalAmplifySession: JournalAuthCapability { - private static var configuration: Result? - - static func configure() throws { - if let configuration { return try configuration.get() } - let result = Result { - let value = try JSONDecoder().decode(AmplifyConfiguration.self, from: Data(#""" - { - "auth": { - "plugins": { - "awsCognitoAuthPlugin": { - "CognitoUserPool": { - "Default": { - "PoolId": "us-east-1_dtagLnju8", - "AppClientId": "69cs1lgme7p8kbgld8n5kseii6", - "Region": "us-east-1" - } - }, - "Auth": { - "Default": { "authenticationFlowType": "USER_SRP_AUTH" } - } - } - } - } - } - """#.utf8)) - try Amplify.add(plugin: AWSCognitoAuthPlugin()) - try Amplify.configure(value) - } - configuration = result - try result.get() - } - - func currentUserID() async throws -> String? { - try Self.configure() - do { - let session = try await Amplify.Auth.fetchAuthSession() - guard session.isSignedIn else { return nil } - return try await Amplify.Auth.getCurrentUser().userId - } catch AuthError.signedOut { - return nil - } - } - - func freshIDToken() async throws -> String { - try Self.configure() - do { - let session = try await Amplify.Auth.fetchAuthSession() - guard session.isSignedIn else { throw JournalPlatformServices.Failure.authenticationRequired } - guard let cognito = session as? AWSAuthCognitoSession else { - throw JournalPlatformServices.Failure.unavailable - } - return try cognito.getCognitoTokens().get().idToken - } catch AuthError.signedOut { - throw JournalPlatformServices.Failure.authenticationRequired - } - } - - func signOut() async throws { - try Self.configure() - guard let result = await Amplify.Auth.signOut() as? AWSCognitoSignOutResult else { - throw JournalPlatformServices.Failure.unavailable - } - switch result { - case .complete, .partial: return - case .failed(let error): throw error - } - } -} diff --git a/swift/JournalApplicationPlatform.swift b/swift/JournalApplicationPlatform.swift index a971ec8..bfd7631 100644 --- a/swift/JournalApplicationPlatform.swift +++ b/swift/JournalApplicationPlatform.swift @@ -1,4 +1,3 @@ -import Amplify import Foundation import Observation #if os(iOS) @@ -58,7 +57,7 @@ import AppKit @ObservationIgnored private var connection = UUID() @ObservationIgnored private var refresh: Task? @ObservationIgnored private var pending = JournalPlatformEvents() - @ObservationIgnored private var authListener: UnsubscribeToken? + @ObservationIgnored private var authListener: NSObjectProtocol? @ObservationIgnored private var shutdown: JournalApplicationShutdown? @ObservationIgnored private var lastEnvironment: JournalEnvironmentSample? @ObservationIgnored private(set) var connected = false @@ -89,7 +88,7 @@ import AppKit func disconnect() { guard connected || runtime != nil else { return } - if let authListener { Amplify.Hub.removeListener(authListener) } + if let authListener { NotificationCenter.default.removeObserver(authListener) } authListener = nil connection = UUID() refresh?.cancel() @@ -233,14 +232,12 @@ import AppKit } private func observeAuthentication() { - if let authListener { Amplify.Hub.removeListener(authListener) } + if let authListener { NotificationCenter.default.removeObserver(authListener) } let identity = connection - authListener = Amplify.Hub.listen(to: .auth) { @Sendable [weak self] payload in - guard [HubPayload.EventName.Auth.signedIn, HubPayload.EventName.Auth.signedOut, - HubPayload.EventName.Auth.sessionExpired, HubPayload.EventName.Auth.userDeleted] - .contains(payload.eventName) else { return } + authListener = NotificationCenter.default.addObserver( + forName: JournalCognitoSession.changed, object: nil, queue: .main) { @Sendable [weak self] _ in Task { @MainActor [weak self] in - guard let self, connection == identity else { return } + guard let self, connection == identity, connected else { return } refreshAuthentication() } } diff --git a/swift/JournalAssetImport.swift b/swift/JournalAssetImport.swift index 0812f09..ff719ac 100644 --- a/swift/JournalAssetImport.swift +++ b/swift/JournalAssetImport.swift @@ -1,3 +1,4 @@ +import CryptoKit import LUIAppleBackend import Observation import PhotosUI @@ -7,11 +8,32 @@ import UniformTypeIdentifiers import UIKit #endif +private actor JournalAssetStaging { + static let shared = JournalAssetStaging() + + func copy(_ source: URL) -> URL? { + let scoped = source.startAccessingSecurityScopedResource() + defer { if scoped { source.stopAccessingSecurityScopedResource() } } + let ext = source.pathExtension.lowercased() + let name = "journal-import-" + UUID().uuidString.lowercased() + (ext.isEmpty ? "" : "." + ext) + let dest = FileManager.default.temporaryDirectory.appendingPathComponent(name) + do { try FileManager.default.copyItem(at: source, to: dest); return dest } + catch { try? FileManager.default.removeItem(at: dest); return nil } + } + + func write(_ data: Data, ext: String) -> URL? { + let dest = FileManager.default.temporaryDirectory.appendingPathComponent("journal-import-" + UUID().uuidString.lowercased() + "." + ext) + do { try data.write(to: dest); return dest } + catch { try? FileManager.default.removeItem(at: dest); return nil } + } +} + @MainActor enum JournalAssetImport { - struct Request: Decodable { + struct Request: Decodable, Equatable { let id: Int let source: String? let staged: Bool? + let maxSelections: Int? } struct PendingItem: Decodable, Identifiable { @@ -19,6 +41,7 @@ import UIKit let path: String let title: String let type: String? + let sourceIdentity: String? var id: String { token } var fileType: String { type ?? "bin" } var isImage: Bool { JournalImportThumbnail.imageTypes.contains(fileType) } @@ -52,58 +75,6 @@ import UIKit } } - /// One pending-attachment chip: thumbnail (images only) over a file icon, - /// with a remove affordance that emits the extension's `remove` event. - private struct PendingCell: SwiftUI.View { - let item: PendingItem - let onRemove: () -> Void - @State private var image: CGImage? - @State private var decodeFailed = false - - var body: some SwiftUI.View { - VStack(spacing: 2) { - Group { - if let image { - Image(decorative: image, scale: 1).resizable().scaledToFill() - } else { - Image(systemName: item.isImage && !decodeFailed ? "photo" : "doc") - .font(.title3) - .foregroundStyle(.secondary) - } - } - .frame(width: 48, height: 48) - .background(.quaternary, in: RoundedRectangle(cornerRadius: 8)) - .clipShape(RoundedRectangle(cornerRadius: 8)) - Text(item.title) - .font(.caption2) - .foregroundStyle(.secondary) - .lineLimit(1) - .truncationMode(.middle) - .frame(width: 56) - } - .overlay(alignment: .topTrailing) { - Button(action: onRemove) { - Image(systemName: "xmark.circle.fill") - .font(.caption) - .foregroundStyle(.secondary) - } - .buttonStyle(.plain) - .padding(2) - .accessibilityIdentifier("journal-asset-remove:" + item.token) - } - .task(id: item.path) { - guard item.isImage else { return } - image = nil - decodeFailed = false - let decoded = await JournalImportThumbnailDecoder.shared.load(item.path) - guard !Task.isCancelled else { return } - image = decoded - decodeFailed = decoded == nil - } - .accessibilityIdentifier("journal-asset-pending:" + item.token) - } - } - #if os(iOS) private struct CameraPicker: UIViewControllerRepresentable { let onImage: (UIImage) -> Void @@ -155,7 +126,10 @@ import UIKit @State private var filePresented = false @State private var photosPresented = false @State private var cameraPresented = false - @State private var photoItem: PhotosPickerItem? + @State private var photoItems: [PhotosPickerItem] = [] + @State private var importTask: Task? + @State private var armedRequest: Request? + @State private var epoch = 0 @State private var handled = false @State private var lastRequest = 0 @State private var error: String? @@ -167,201 +141,196 @@ import UIKit private var request: Request? { properties?.request } private func emit(_ object: [String: Any]) -> Bool { - guard let data = try? JSONSerialization.data(withJSONObject: object) - else { return false } + guard let data = try? JSONSerialization.data(withJSONObject: object) else { return false } return JournalExtensions.emit(context: context, payload: data) } + private func isCurrent(_ request: Request, epoch: Int) -> Bool { + self.epoch == epoch && self.request?.id == request.id + && context.isUserInteractionEnabled && properties?.enabled == true + } + private func emitDismissed() { guard !handled else { return } handled = true - _ = emit(["action": "dismissed"]) + _ = emit(["action": "dismissed", "request": armedRequest?.id ?? 0]) } - private func emitUnavailable(_ reason: String) { + private func emitUnavailable(_ reason: String, request: Request) { handled = true - _ = emit([ - "action": "unavailable", - "reason": reason, - "request": request?.id ?? 0, - ]) + _ = emit(["action": "unavailable", "reason": reason, "request": request.id]) } - private func emitRemove(_ token: String) { - _ = emit(["action": "remove", "token": token]) + private func requestPayload(_ request: Request) -> [String: Any] { + ["id": request.id, "source": request.source ?? "files", "staged": request.staged ?? false] } - private func emitPick(path: String, title: String, type: String, retained: URL?) { - handled = true - let operation = UUID().uuidString.lowercased() - if let retained { selection.retain(retained, operation: operation) } - let ok = emit([ - "operation": operation, - "asset": UUID().uuidString.lowercased(), - "localMutation": UUID().uuidString.lowercased(), - "metadataMutation": UUID().uuidString.lowercased(), - "path": path, - "title": title, - "type": type.isEmpty ? "bin" : type, - "request": [ - "id": request?.id ?? 0, - "source": request?.source ?? "files", - "staged": request?.staged ?? false, - ], - ] as [String: Any]) - if !ok { - selection.release() - error = "The destination is no longer available. Select the file again." - } + private func pick(path: String, title: String, type: String, identity: String) -> [String: Any] { + ["operation": UUID().uuidString.lowercased(), "asset": UUID().uuidString.lowercased(), + "localMutation": UUID().uuidString.lowercased(), "metadataMutation": UUID().uuidString.lowercased(), + "path": path, "title": title, "type": type.isEmpty ? "bin" : type, "sourceIdentity": identity] } - /// Staged requests copy the pick into a temp file so the path stays valid - /// after the picker's security scope is released (attach-on-save). - private func stagedCopy(of url: URL, title: String) -> (path: String, title: String)? { - let ext = url.pathExtension.lowercased() - let name = - "journal-import-" + UUID().uuidString.lowercased() - + (ext.isEmpty ? "" : "." + ext) - let dest = FileManager.default.temporaryDirectory.appendingPathComponent(name) - try? FileManager.default.removeItem(at: dest) - do { - try FileManager.default.copyItem(at: url, to: dest) - return (dest.path(percentEncoded: false), title) - } catch { - return nil + private func discard(_ items: [[String: Any]]) { + for item in items { + guard let path = item["path"] as? String else { continue } + try? FileManager.default.removeItem(atPath: path) } } - private func stageData(_ data: Data, ext: String, title: String) { - let dest = FileManager.default.temporaryDirectory - .appendingPathComponent("journal-import-" + UUID().uuidString.lowercased() + "." + ext) - do { - try data.write(to: dest) - emitPick(path: dest.path(percentEncoded: false), title: title, type: ext, retained: nil) - } catch { - self.error = "Unable to save the image. Please try again." + private func emitBatch(_ items: [[String: Any]], failures: Int, request: Request, epoch: Int) { + guard !Task.isCancelled, isCurrent(request, epoch: epoch) else { discard(items); return } + handled = true + var payload: [String: Any] = ["action": "picked-batch", "request": requestPayload(request), "items": items] + if failures > 0 { payload["error"] = "\(failures) selected item(s) could not be added. Your other attachments are kept." } + if !emit(payload) { + discard(items) + error = "The destination is no longer available. Select the files again." } } - private func handleFilePick(_ source: URL) { - let title = source.lastPathComponent - let type = source.pathExtension.lowercased() - if request?.staged == true { - selection.retain(source, operation: UUID().uuidString.lowercased()) - let staged = stagedCopy(of: source, title: title) - selection.release() - guard let staged else { - self.error = "Unable to access the selected file. Please try again." - return + private func handleFiles(_ sources: [URL], request: Request, epoch: Int) async { + guard !Task.isCancelled, isCurrent(request, epoch: epoch) else { return } + handled = true + guard request.staged == true else { + guard let source = sources.first else { return } + var item = pick(path: source.path(percentEncoded: false), title: source.lastPathComponent, + type: source.pathExtension.lowercased(), identity: "files:" + source.standardizedFileURL.absoluteString) + let operation = item["operation"] as! String + selection.retain(source, operation: operation) + item["request"] = requestPayload(request) + if !emit(item) { selection.release(); error = "The destination is no longer available. Select the file again." } + return + } + var items: [[String: Any]] = [] + var seen = Set((properties?.pending ?? []).compactMap(\.sourceIdentity)) + var failures = 0 + let remaining = max(0, request.maxSelections ?? sources.count) + for source in sources { + let identity = "files:" + source.standardizedFileURL.absoluteString + guard seen.insert(identity).inserted else { continue } + guard items.count < remaining else { failures += 1; continue } + guard !Task.isCancelled, isCurrent(request, epoch: epoch) else { discard(items); return } + guard let copy = await JournalAssetStaging.shared.copy(source) else { failures += 1; continue } + guard !Task.isCancelled, isCurrent(request, epoch: epoch) else { + try? FileManager.default.removeItem(at: copy); discard(items); return } - emitPick(path: staged.path, title: staged.title, type: type, retained: nil) - } else { - emitPick( - path: source.path(percentEncoded: false), title: title, type: type, - retained: source) + items.append(pick(path: copy.path(percentEncoded: false), title: source.lastPathComponent, + type: source.pathExtension.lowercased(), identity: identity)) } + emitBatch(items, failures: failures, request: request, epoch: epoch) } - private func importPhoto(_ item: PhotosPickerItem) async { - photoItem = nil - guard let data = try? await item.loadTransferable(type: Data.self) - else { - self.error = "Unable to read the selected photo. Please try again." - return + private func stageData(_ data: Data, ext: String, title: String, identity: String) async -> [String: Any]? { + guard let dest = await JournalAssetStaging.shared.write(data, ext: ext) else { return nil } + return pick(path: dest.path(percentEncoded: false), title: title, type: ext, identity: identity) + } + + private func importPhotos(_ selected: [PhotosPickerItem], request: Request, epoch: Int) async { + var items: [[String: Any]] = [] + var seen = Set((properties?.pending ?? []).compactMap(\.sourceIdentity)) + var failures = 0 + let remaining = max(0, request.maxSelections ?? selected.count) + for item in selected { + guard !Task.isCancelled, isCurrent(request, epoch: epoch) else { discard(items); return } + if let identifier = item.itemIdentifier, seen.contains("photos:" + identifier) { continue } + guard items.count < remaining else { failures += 1; continue } + guard let data = try? await item.loadTransferable(type: Data.self) else { failures += 1; continue } + guard !Task.isCancelled, isCurrent(request, epoch: epoch) else { discard(items); return } + let identity = "photos:" + (item.itemIdentifier ?? SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined()) + guard seen.insert(identity).inserted else { continue } + let ext = item.supportedContentTypes.first?.preferredFilenameExtension ?? "jpg" + if let staged = await stageData(data, ext: ext, title: "photo." + ext, identity: identity) { items.append(staged) } + else { failures += 1 } } - let ext = - item.supportedContentTypes.first?.preferredFilenameExtension ?? "jpg" - stageData(data, ext: ext, title: "photo." + ext) + emitBatch(items, failures: failures, request: request, epoch: epoch) } var body: some SwiftUI.View { - VStack(spacing: 8) { - if let pending = properties?.pending, !pending.isEmpty { - ScrollView(.horizontal, showsIndicators: false) { - HStack(spacing: 12) { - ForEach(pending) { item in - PendingCell(item: item) { emitRemove(item.token) } - } - } - .padding(.horizontal, 4) - } - .frame(height: 72) - } + let _ = context.revision + // Composer owns the attachment cards; this extension only owns picking. + VStack(spacing: 0) { context.content + Color.clear.frame(width: 1, height: 1).allowsHitTesting(false) } - .fileImporter( - isPresented: $filePresented, allowedContentTypes: [.item], - allowsMultipleSelection: false - ) { result in - guard context.isUserInteractionEnabled else { return } - do { - guard let source = try result.get().first else { - emitDismissed() - return + .fileImporter(isPresented: $filePresented, allowedContentTypes: [.item], allowsMultipleSelection: armedRequest?.staged == true) { result in + guard let request = armedRequest else { return } + let epoch = self.epoch + guard isCurrent(request, epoch: epoch) else { return } + switch result { + case .success(let sources): + if sources.isEmpty { emitDismissed() } + else { + handled = true + importTask?.cancel() + importTask = Task { await handleFiles(sources, request: request, epoch: epoch) } } - handleFilePick(source) - } catch { + case .failure(let failure): selection.release() - if (error as NSError).code == NSUserCancelledError { - emitDismissed() - } else { - self.error = "Unable to access the selected file. Please try again." - } + if (failure as NSError).code == NSUserCancelledError { emitDismissed() } + else if isCurrent(request, epoch: epoch) { emitUnavailable("Unable to access the selected files. Please try again.", request: request) } } } - .photosPicker( - isPresented: $photosPresented, selection: $photoItem, matching: .images - ) + .photosPicker(isPresented: $photosPresented, selection: $photoItems, + maxSelectionCount: max(1, armedRequest?.maxSelections ?? 1), selectionBehavior: .ordered, matching: .images) .sheet(isPresented: $cameraPresented) { #if os(iOS) - CameraPicker( - onImage: { image in + CameraPicker(onImage: { image in + guard let request = armedRequest, isCurrent(request, epoch: epoch) else { return } + handled = true + let epoch = self.epoch + importTask?.cancel() + importTask = Task { + let item: [String: Any]? if let data = image.jpegData(compressionQuality: 0.9) { - stageData(data, ext: "jpg", title: "camera.jpg") - } else { - self.error = "Unable to save the image. Please try again." - } - }, - onCancel: emitDismissed) + item = await stageData(data, ext: "jpg", title: "camera.jpg", identity: "camera:" + UUID().uuidString.lowercased()) + } else { item = nil } + emitBatch(item.map { [$0] } ?? [], failures: item == nil ? 1 : 0, request: request, epoch: epoch) + } + }, onCancel: emitDismissed) #else EmptyView() #endif } - .onChange(of: filePresented) { _, isPresented in - if isPresented { - handled = false - } - } - .onChange(of: photosPresented) { _, isPresented in - if isPresented { handled = false } - } - .onChange(of: photoItem) { _, item in - guard let item else { return } - Task { await importPhoto(item) } + .onChange(of: filePresented) { _, presented in if !presented { emitDismissed() } } + .onChange(of: photosPresented) { _, presented in if !presented && photoItems.isEmpty { emitDismissed() } } + .onChange(of: photoItems) { _, items in + guard !items.isEmpty, let request = armedRequest else { return } + guard isCurrent(request, epoch: epoch) else { return } + handled = true + photoItems = [] + importTask?.cancel() + let epoch = self.epoch + importTask = Task { await importPhotos(items, request: request, epoch: epoch) } } - .onChange(of: request?.id) { _, id in - guard let id, id != lastRequest, context.isUserInteractionEnabled, - properties?.enabled == true, selection.operation == nil - else { return } + .onChange(of: request?.id, initial: true) { _, id in + guard let id, id != lastRequest else { return } + epoch += 1 + importTask?.cancel() + let wasPresenting = filePresented || photosPresented || cameraPresented + if id <= 0 || wasPresenting { + handled = true + filePresented = false; photosPresented = false; cameraPresented = false + armedRequest = nil + lastRequest = id + return + } + guard context.isUserInteractionEnabled, properties?.enabled == true, selection.operation == nil, let request else { return } lastRequest = id + armedRequest = request handled = false - switch request?.source ?? "files" { - case "files": - filePresented = true - case "photos": - photosPresented = true + photoItems = [] + switch request.source ?? "files" { + case "files": filePresented = true + case "photos": photosPresented = true case "camera": #if os(iOS) - if UIImagePickerController.isSourceTypeAvailable(.camera) { - cameraPresented = true - } else { - emitUnavailable("The camera is not available on this device.") - } + if UIImagePickerController.isSourceTypeAvailable(.camera) { cameraPresented = true } + else { emitUnavailable("The camera is not available on this device.", request: request) } #else - emitUnavailable("Camera capture is not available on this platform.") + emitUnavailable("Camera capture is not available on this platform.", request: request) #endif - default: - emitUnavailable("Unknown attachment source.") + default: emitUnavailable("Unknown attachment source.", request: request) } } .onChange(of: properties?.completion) { _, operation in @@ -369,11 +338,12 @@ import UIKit selection.release() error = properties?.error } - .onDisappear { selection.release() } - .alert( - "Unable to import file", - isPresented: Binding(get: { error != nil }, set: { if !$0 { error = nil } }) - ) { + .onDisappear { + epoch += 1 + importTask?.cancel() + selection.release() + } + .alert("Unable to import file", isPresented: Binding(get: { error != nil }, set: { if !$0 { error = nil } })) { Button("OK", role: .cancel) { error = nil } } message: { Text(error ?? "") } } diff --git a/swift/JournalAuthentication.swift b/swift/JournalAuthentication.swift index 8e06012..c7f33fa 100644 --- a/swift/JournalAuthentication.swift +++ b/swift/JournalAuthentication.swift @@ -1,129 +1,35 @@ import Foundation import Observation -/// Native authentication presentation only; account and graph decisions stay in OCaml. -enum JournalAuthenticationStep: Equatable { - struct Challenge: Equatable { - let title: String - let prompt: String - enum Input: Equatable { case text, oneTimeCode, password, newPassword, emailAddress } - var input = Input.text - var detail: String? = nil - var choices: [String] = [] - } - case signIn, resetRequest, resetConfirmation(String), confirmAccount(String) - case challenge(Challenge) - case complete -} - -enum JournalAuthenticationCommand: Equatable { - case signIn(username: String, password: String) - case confirm(String) - case requestReset(String) - case confirmReset(username: String, code: String, password: String) - case confirmAccount(username: String, code: String) - case resendAccount(String) -} - @MainActor protocol JournalAuthenticationAPI { - func perform(_ command: JournalAuthenticationCommand) async throws -> JournalAuthenticationStep + func signIn() async throws + func cancelSignIn() } - @Observable @MainActor final class JournalAuthentication { - var username = "" - var password = "" - var confirmation = "" - var newPassword = "" - private(set) var step = JournalAuthenticationStep.signIn + private(set) var complete = false private(set) var busy = false private(set) var message: String? @ObservationIgnored private let api: any JournalAuthenticationAPI @ObservationIgnored private var revision: UInt64 = 0 - init(api: any JournalAuthenticationAPI) { self.api = api } - private var user: String { username.trimmingCharacters(in: .whitespacesAndNewlines) } - - var canSubmit: Bool { - guard !busy else { return false } - switch step { - case .signIn: return !user.isEmpty && !password.isEmpty - case .resetRequest: return !user.isEmpty - case .resetConfirmation: return !user.isEmpty && !confirmation.isEmpty && !newPassword.isEmpty - case .confirmAccount: return !user.isEmpty && !confirmation.isEmpty - case .challenge(let challenge): - return !confirmation.isEmpty && (challenge.choices.isEmpty || challenge.choices.contains(confirmation)) - case .complete: return false - } - } - + var canSubmit: Bool { !busy && !complete } func submit() async { guard canSubmit else { return } - let command: JournalAuthenticationCommand - switch step { - case .signIn: command = .signIn(username: user, password: password) - case .resetRequest: command = .requestReset(user) - case .resetConfirmation: command = .confirmReset(username: user, code: confirmation, password: newPassword) - case .confirmAccount: command = .confirmAccount(username: user, code: confirmation) - case .challenge: command = .confirm(confirmation) - case .complete: return - } - await perform(command) - } - - func forgotPassword() { - guard !busy else { return } - clearSecrets() - message = nil - step = .resetRequest - } - - func back() { - guard !busy else { return } - clearSecrets() - message = nil - step = .signIn - } - - func resend() async { - guard !busy, case .confirmAccount = step, !user.isEmpty else { return } - await perform(.resendAccount(user)) - } - - func cancel() { - revision &+= 1 - clearSecrets() - message = nil - step = .signIn - } - - private func perform(_ command: JournalAuthenticationCommand) async { - busy = true - message = nil - let current = revision - defer { busy = false } + let version = revision + busy = true; message = nil + defer { if revision == version { busy = false } } do { - let next = try await api.perform(command) + try await api.signIn() try Task.checkCancellation() - guard revision == current else { return } - clearSecrets() - step = next - if next == .signIn { - switch command { - case .confirmReset: message = "Password updated. Sign in with your new password." - case .confirmAccount: message = "Account confirmed. Sign in to continue." - default: break - } - } - } catch is CancellationError { - if revision == current { clearSecrets() } - } catch { - if revision == current { message = error.localizedDescription } + if revision == version { complete = true } + } catch is CancellationError { } + catch is URLError { + if revision == version { message = "Unable to connect to sign-in. Please try again." } } + catch { if revision == version { message = error.localizedDescription } } } - - private func clearSecrets() { - password = "" - confirmation = "" - newPassword = "" + func cancel() { + revision &+= 1; api.cancelSignIn() + busy = false; message = nil; complete = false } } diff --git a/swift/JournalAuthenticationView.swift b/swift/JournalAuthenticationView.swift index c0d473a..2d99061 100644 --- a/swift/JournalAuthenticationView.swift +++ b/swift/JournalAuthenticationView.swift @@ -3,183 +3,39 @@ import SwiftUI struct JournalAuthenticationView: View { @State private var model: JournalAuthentication @State private var operation: Task? - private enum Field: Hashable { case username, password, confirmation, newPassword } - @FocusState private var focusedField: Field? let authenticated: () -> Void - init(api: any JournalAuthenticationAPI, authenticated: @escaping () -> Void) { _model = State(initialValue: JournalAuthentication(api: api)) self.authenticated = authenticated } - var body: some View { ScrollView { VStack(alignment: .leading, spacing: 20) { - Text(title) - .font(.title).bold() - .fixedSize(horizontal: false, vertical: true) - .accessibilityAddTraits(.isHeader) - fields + Text("Sign in to Logseq").font(.title).bold().accessibilityAddTraits(.isHeader) + Text("Continue to the secure sign-in page to sign in or recover your account.") + .foregroundStyle(.secondary) if let message = model.message { - Text(message).foregroundStyle(.primary).accessibilityIdentifier("authentication-message") - } - if model.busy { - ProgressView("Please wait…").accessibilityIdentifier("authentication-progress") + Text(message).foregroundStyle(.red).accessibilityIdentifier("authentication-error") } - Button(submitLabel) { submit() } - .buttonStyle(.borderedProminent) - .disabled(!model.canSubmit) - .keyboardShortcut(.defaultAction) - .accessibilityIdentifier("authentication-submit") - if model.step == .signIn { - Button("Forgot password?") { model.forgotPassword() } - .buttonStyle(.bordered).disabled(model.busy) - } else if model.step != .complete { - Button("Back to sign in") { model.back() } - .buttonStyle(.bordered).disabled(model.busy) + Button { + operation = Task { + await model.submit() + if model.complete && !Task.isCancelled { authenticated() } + } + } label: { + HStack { + if model.busy { ProgressView() } + Text(model.busy ? "Opening sign-in…" : "Continue to sign in") + }.frame(maxWidth: .infinity) } - } - .textFieldStyle(.roundedBorder) - .frame(maxWidth: 440, alignment: .leading) - .padding(24) - .frame(maxWidth: .infinity) - .tint(.primary) - .disabled(model.busy) - } - .navigationTitle("Account") - #if os(iOS) - .navigationBarTitleDisplayMode(.inline) - #endif - .task(id: model.step) { focusFirstField() } - .onDisappear { operation?.cancel(); model.cancel() } - .onChange(of: model.step) { _, step in - if step == .complete { authenticated() } - } - } - - @ViewBuilder private var fields: some View { - switch model.step { - case .signIn: - usernameField.submitLabel(.next).onSubmit { focusedField = .password } - SecureField("Password", text: $model.password).textContentType(.password) - .focused($focusedField, equals: .password) - .submitLabel(.go).onSubmit { submit() } - .accessibilityIdentifier("authentication-password") - case .resetRequest: - usernameField.submitLabel(.go).onSubmit { submit() } - case .resetConfirmation(let destination): - Text("Enter the code sent to \(destination).") - codeField("Verification code").submitLabel(.next) - .focused($focusedField, equals: .confirmation) - .onSubmit { focusedField = .newPassword } - SecureField("New password", text: $model.newPassword).textContentType(.newPassword) - .focused($focusedField, equals: .newPassword) - .submitLabel(.go).onSubmit { submit() } - case .confirmAccount(let destination): - Text("Enter the confirmation code sent to \(destination).") - codeField("Confirmation code").submitLabel(.go).onSubmit { submit() } - .focused($focusedField, equals: .confirmation) - Button("Resend code") { operation = Task { await model.resend() } } - .buttonStyle(.bordered).disabled(model.busy) - case .challenge(let challenge): - if let detail = challenge.detail { Text(detail).textSelection(.enabled) } - if !challenge.choices.isEmpty { - Picker(challenge.prompt, selection: $model.confirmation) { - Text("Choose a method").tag("") - ForEach(challenge.choices, id: \.self) { value in Text(Self.methodName(value)).tag(value) } + .buttonStyle(.borderedProminent).disabled(!model.canSubmit) + .accessibilityIdentifier("authentication-submit") + if model.busy { + Button("Cancel", role: .cancel) { cancel() } } - } else { - challengeField(challenge) - .focused($focusedField, equals: .confirmation) - .submitLabel(.go).onSubmit { submit() } - .accessibilityIdentifier("authentication-challenge") - } - case .complete: ProgressView("Opening your account") - } - } - - @ViewBuilder private func challengeField(_ challenge: JournalAuthenticationStep.Challenge) -> some View { - switch challenge.input { - case .password, .newPassword: - SecureField(challenge.prompt, text: $model.confirmation) - .textContentType(challenge.input == .newPassword ? .newPassword : .password) - case .oneTimeCode: - codeField(challenge.prompt) - case .emailAddress: - TextField(challenge.prompt, text: $model.confirmation) - .textContentType(.emailAddress) - .autocorrectionDisabled() - #if os(iOS) - .textInputAutocapitalization(.never).keyboardType(.emailAddress) - #endif - case .text: - TextField(challenge.prompt, text: $model.confirmation) - .autocorrectionDisabled() - #if os(iOS) - .textInputAutocapitalization(.never) - #endif - } - } - - private var usernameField: some View { - TextField("Username", text: $model.username).textContentType(.username) - .autocorrectionDisabled().focused($focusedField, equals: .username) - .accessibilityIdentifier("authentication-username") - #if os(iOS) - .textInputAutocapitalization(.never) - #endif - } - - private func codeField(_ title: String) -> some View { - TextField(title, text: $model.confirmation).textContentType(.oneTimeCode) - .autocorrectionDisabled() - #if os(iOS) - .textInputAutocapitalization(.never).keyboardType(.asciiCapable) - #endif - } - - private func focusFirstField() { - switch model.step { - case .signIn: focusedField = model.username.isEmpty ? .username : .password - case .resetRequest: focusedField = .username - case .resetConfirmation, .confirmAccount: focusedField = .confirmation - case .challenge(let challenge): focusedField = challenge.choices.isEmpty ? .confirmation : nil - case .complete: focusedField = nil - } - } - - private var title: String { - switch model.step { - case .signIn: return "Sign in to Logseq" - case .resetRequest, .resetConfirmation: return "Reset your password" - case .confirmAccount: return "Confirm your account" - case .challenge(let challenge): return challenge.title - case .complete: return "Signed in" - } - } - - private var submitLabel: String { - switch model.step { - case .signIn: return "Sign in" - case .resetRequest: return "Send reset code" - case .resetConfirmation: return "Update password" - default: return "Continue" - } - } - - private func submit() { - guard model.canSubmit else { return } - operation = Task { await model.submit() } - } - - private static func methodName(_ value: String) -> String { - switch value { - case "SMS_MFA", "SMS_OTP": return "Text message" - case "SOFTWARE_TOKEN_MFA": return "Authenticator app" - case "EMAIL_OTP": return "Email" - case "PASSWORD", "PASSWORD_SRP": return "Password" - case "WEB_AUTHN": return "Passkey" - default: return "Verification" + }.padding(24).frame(maxWidth: 480, alignment: .leading) } + .onDisappear { cancel() } } + private func cancel() { operation?.cancel(); operation = nil; model.cancel() } } diff --git a/swift/JournalCognitoNative.swift b/swift/JournalCognitoNative.swift new file mode 100644 index 0000000..3a20c80 --- /dev/null +++ b/swift/JournalCognitoNative.swift @@ -0,0 +1,142 @@ +import AuthenticationServices +import Foundation +import Security +#if os(iOS) +import UIKit +#else +import AppKit +#endif + +/// One queue preserves save/clear order and keeps blocking Security calls off the UI actor. +private final class JournalCognitoKeychain: Sendable { + private let queue = DispatchQueue(label: "com.logseq.journal.oauth-storage") + private func perform(_ operation: @escaping @Sendable () throws -> T) async throws -> T { + try await withCheckedThrowingContinuation { continuation in + queue.async { continuation.resume(with: Result(catching: operation)) } + } + } + private static var query: [CFString: Any] { + [kSecClass: kSecClassGenericPassword, kSecAttrService: "com.logseq.journal.cognito.tokens", + kSecAttrAccount: "current-session", kSecUseDataProtectionKeychain: true, + kSecAttrSynchronizable: false] + } + private static func clearLegacySession() throws { + // Exact key from the removed SDK's AWSCognitoAuthCredentialStore. No token decoding/fallback. + let legacy: [CFString: Any] = [kSecClass: kSecClassGenericPassword, + kSecAttrService: "com.amplify.awsCognitoAuthPlugin", + kSecAttrAccount: "amplify.us-east-1_dtagLnju8.session", kSecUseDataProtectionKeychain: true] + let status = SecItemDelete(legacy as CFDictionary) + guard status == errSecSuccess || status == errSecItemNotFound else { throw JournalCognitoError.storage } + } + func load() async throws -> JournalCognitoTokens? { + try await perform { + var query = Self.query + query[kSecReturnData] = true; query[kSecMatchLimit] = kSecMatchLimitOne + var item: CFTypeRef? + let status = SecItemCopyMatching(query as CFDictionary, &item) + if status == errSecItemNotFound { return nil } + guard status == errSecSuccess, let data = item as? Data, data.count <= 131072 + else { throw JournalCognitoError.storage } + return try JSONDecoder().decode(JournalCognitoTokens.self, from: data) + } + } + func save(_ tokens: JournalCognitoTokens) async throws { + try await perform { + let data = try JSONEncoder().encode(tokens) + guard data.count <= 131072 else { throw JournalCognitoError.storage } + let status = SecItemUpdate(Self.query as CFDictionary, [kSecValueData: data] as CFDictionary) + if status == errSecItemNotFound { + var query = Self.query + query[kSecValueData] = data; query[kSecAttrAccessible] = kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly + guard SecItemAdd(query as CFDictionary, nil) == errSecSuccess else { throw JournalCognitoError.storage } + } else if status != errSecSuccess { throw JournalCognitoError.storage } + } + } + func clear() async throws { + try await perform { + let status = SecItemDelete(Self.query as CFDictionary) + guard status == errSecSuccess || status == errSecItemNotFound else { throw JournalCognitoError.storage } + } + } + func retireLegacy() async throws { try await perform { try Self.clearLegacySession() } } +} + +/// Cancels and resumes exactly once, including late callbacks, start failure and task cancellation. +@MainActor final class JournalCognitoWebSession: NSObject, ASWebAuthenticationPresentationContextProviding { + private var operation: UUID? + private var session: ASWebAuthenticationSession? + private var continuation: CheckedContinuation? + private var timer: Task? + private var anchor: ASPresentationAnchor? + func authenticate(_ url: URL, scheme: String) async throws -> URL { + guard operation == nil else { throw JournalCognitoError.unavailable } + #if os(iOS) + anchor = UIApplication.shared.connectedScenes.compactMap { $0 as? UIWindowScene } + .filter { $0.activationState == .foregroundActive }.flatMap(\.windows).first(where: \.isKeyWindow) + #else + anchor = NSApplication.shared.keyWindow ?? NSApplication.shared.mainWindow + #endif + guard anchor != nil else { throw JournalCognitoError.unavailable } + let id = UUID(); operation = id + defer { if operation == id { finish(id, .failure(CancellationError()), cancel: true) } } + return try await withTaskCancellationHandler { + try Task.checkCancellation() + return try await withCheckedThrowingContinuation { continuation in + self.continuation = continuation + let session = ASWebAuthenticationSession(url: url, callbackURLScheme: scheme) { [weak self] callback, error in + Task { @MainActor [weak self] in + if let callback { self?.finish(id, .success(callback)) } + else if (error as? ASWebAuthenticationSessionError)?.code == .canceledLogin { + self?.finish(id, .failure(CancellationError())) + } else { self?.finish(id, .failure(JournalCognitoError.unavailable)) } + } + } + self.session = session + session.presentationContextProvider = self + session.prefersEphemeralWebBrowserSession = true + timer = Task { [weak self] in + do { try await Task.sleep(for: .seconds(300)) } catch { return } + self?.finish(id, .failure(JournalCognitoError.expired), cancel: true) + } + if !session.start() { finish(id, .failure(JournalCognitoError.unavailable), cancel: true) } + } + } onCancel: { Task { @MainActor [weak self] in self?.finish(id, .failure(CancellationError()), cancel: true) } } + } + func cancel() { + if let operation { finish(operation, .failure(CancellationError()), cancel: true) } + } + private func finish(_ id: UUID, _ result: Result, cancel: Bool = false) { + guard operation == id else { return } + let waiter = continuation, active = session + operation = nil; continuation = nil; session = nil; anchor = nil + timer?.cancel(); timer = nil + if cancel { active?.cancel() } + waiter?.resume(with: result) + } + func presentationAnchor(for session: ASWebAuthenticationSession) -> ASPresentationAnchor { anchor ?? ASPresentationAnchor() } +} + +private final class JournalOAuthRedirectPolicy: NSObject, URLSessionTaskDelegate, Sendable { + func urlSession(_ session: URLSession, task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, newRequest request: URLRequest, + completionHandler: @escaping @Sendable (URLRequest?) -> Void) { completionHandler(nil) } +} +@MainActor enum JournalCognitoNative { + static let shared: JournalCognitoSession = { + let storage = JournalCognitoKeychain(), browser = JournalCognitoWebSession() + let configuration = URLSessionConfiguration.ephemeral + configuration.httpCookieStorage = nil; configuration.urlCache = nil + configuration.timeoutIntervalForRequest = 15; configuration.timeoutIntervalForResource = 20 + let http = URLSession(configuration: configuration, delegate: JournalOAuthRedirectPolicy(), delegateQueue: nil) + return JournalCognitoSession(configuration: .production, + storage: JournalCognitoStorage(load: { try await storage.load() }, + save: { try await storage.save($0) }, clear: { try await storage.clear() }, + retireLegacy: { try await storage.retireLegacy() }), + browser: JournalCognitoBrowser(authenticate: { try await browser.authenticate($0, scheme: $1) }, cancel: { browser.cancel() }), + transport: { request in + let (data, response) = try await http.data(for: request) + guard let response = response as? HTTPURLResponse else { throw JournalCognitoError.request } + return (data, response) + }) + }() +} diff --git a/swift/JournalCognitoOAuth.swift b/swift/JournalCognitoOAuth.swift new file mode 100644 index 0000000..52480e0 --- /dev/null +++ b/swift/JournalCognitoOAuth.swift @@ -0,0 +1,185 @@ +import CryptoKit +import Foundation +import Security + +struct JournalCognitoConfiguration: Sendable { + let domain: String + let clientID: String + let userPoolID: String + let region: String + let redirectURI: String + var issuer: String { "https://cognito-idp.\(region).amazonaws.com/\(userPoolID)" } + static let production = Self(domain: "logseq-prod.auth.us-east-1.amazoncognito.com", + clientID: "69cs1lgme7p8kbgld8n5kseii6", userPoolID: "us-east-1_dtagLnju8", + region: "us-east-1", redirectURI: "logseqjournal://auth/callback") +} + +enum JournalCognitoError: Error, LocalizedError { + case configuration, callback, expired, invalidToken, storage, unavailable, request, invalidGrant + var errorDescription: String? { + switch self { + case .configuration: "Web sign-in is not configured." + case .callback: "The sign-in response could not be verified. Please try again." + case .expired: "Sign-in timed out. Please try again." + case .invalidToken: "The account session could not be verified. Please sign in again." + case .storage: "Unable to save the account session securely. Please try again." + case .unavailable: "Unable to open the sign-in page. Please try again." + case .request: "Unable to connect to sign-in. Please try again." + case .invalidGrant: "Your account session expired. Please sign in again." + } + } +} + +enum JournalCognitoOAuth { + static func url64(_ data: Data) -> String { + data.base64EncodedString().replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_").replacingOccurrences(of: "=", with: "") + } + static func data64(_ value: String) throws -> Data { + guard !value.isEmpty, value.utf8.allSatisfy({ + (65...90).contains($0) || (97...122).contains($0) || (48...57).contains($0) || $0 == 45 || $0 == 95 + }) else { throw JournalCognitoError.invalidToken } + let base = value.replacingOccurrences(of: "-", with: "+").replacingOccurrences(of: "_", with: "/") + guard let data = Data(base64Encoded: base + String(repeating: "=", count: (4 - base.count % 4) % 4)) + else { throw JournalCognitoError.invalidToken } + return data + } + static func random() throws -> String { + var bytes = [UInt8](repeating: 0, count: 32) + guard SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes) == errSecSuccess + else { throw JournalCognitoError.unavailable } + return url64(Data(bytes)) + } + static func challenge(_ verifier: String) -> String { url64(Data(SHA256.hash(data: Data(verifier.utf8)))) } + static func form(_ values: [String: String]) -> Data { + let allowed = CharacterSet(charactersIn: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~") + return Data(values.sorted { $0.key < $1.key }.map { + $0.key.addingPercentEncoding(withAllowedCharacters: allowed)! + "=" + + $0.value.addingPercentEncoding(withAllowedCharacters: allowed)! + }.joined(separator: "&").utf8) + } + static func authorization(_ configuration: JournalCognitoConfiguration, state: String, + verifier: String, nonce: String) throws -> URL { + guard !configuration.clientID.isEmpty, !configuration.region.isEmpty, !configuration.userPoolID.isEmpty, + let redirect = URLComponents(string: configuration.redirectURI), redirect.scheme == "logseqjournal", + redirect.host == "auth", redirect.path == "/callback", redirect.query == nil, redirect.fragment == nil, + redirect.user == nil, redirect.password == nil, redirect.port == nil, + configuration.domain.utf8.allSatisfy({ (45...57).contains($0) || (97...122).contains($0) }), + configuration.domain.contains(".") else { throw JournalCognitoError.configuration } + var url = URLComponents() + url.scheme = "https"; url.host = configuration.domain; url.path = "/oauth2/authorize" + url.queryItems = ["response_type": "code", "client_id": configuration.clientID, + "redirect_uri": configuration.redirectURI, "scope": "openid", "state": state, + "code_challenge": challenge(verifier), "code_challenge_method": "S256", "nonce": nonce] + .map { URLQueryItem(name: $0.key, value: $0.value) } + guard let result = url.url else { throw JournalCognitoError.configuration } + return result + } + static func code(_ callback: URL, configuration: JournalCognitoConfiguration, state: String) throws -> String { + guard let actual = URLComponents(url: callback, resolvingAgainstBaseURL: false), + let expected = URLComponents(string: configuration.redirectURI), + actual.scheme == expected.scheme, actual.host == expected.host, + actual.percentEncodedPath == expected.percentEncodedPath, + actual.port == nil, actual.user == nil, actual.password == nil, actual.fragment == nil + else { throw JournalCognitoError.callback } + var values: [String: String] = [:] + for item in actual.queryItems ?? [] { + guard values[item.name] == nil, let value = item.value else { throw JournalCognitoError.callback } + values[item.name] = value + } + guard values["state"] == state, values["error"] == nil, + let code = values["code"], !code.isEmpty, code.utf8.count <= 8192 else { throw JournalCognitoError.callback } + return code + } +} + +struct JournalCognitoTokens: Codable, Sendable { + let idToken: String + let accessToken: String + let refreshToken: String + let userID: String + let issuer: String + let clientID: String + let expiresAt: Date +} + +struct JournalCognitoTokenResponse: Decodable { + let id_token: String + let access_token: String + let refresh_token: String? + let expires_in: Int + let token_type: String +} + +/// Verify both signatures against keys from the fixed user-pool issuer, never a JWT URL. +struct JournalCognitoTokenValidator { + struct Key: Decodable { + let kty: String; let kid: String; let alg: String?; let use: String?; let n: String; let e: String + } + struct Keys: Decodable { let keys: [Key] } + let keys: Keys + init(jwks: Data) throws { + guard jwks.count <= 262144 else { throw JournalCognitoError.invalidToken } + keys = try JSONDecoder().decode(Keys.self, from: jwks) + } + private func claims(_ token: String) throws -> [String: Any] { + guard token.utf8.count <= 32768 else { throw JournalCognitoError.invalidToken } + let parts = token.split(separator: ".", omittingEmptySubsequences: false).map(String.init) + guard parts.count == 3, + let header = try JSONSerialization.jsonObject(with: JournalCognitoOAuth.data64(parts[0])) as? [String: Any], + header["alg"] as? String == "RS256", header["crit"] == nil, + let kid = header["kid"] as? String, + keys.keys.filter({ $0.kid == kid }).count == 1, + let key = keys.keys.first(where: { $0.kid == kid }), key.kty == "RSA", + key.alg == nil || key.alg == "RS256", key.use == nil || key.use == "sig" + else { throw JournalCognitoError.invalidToken } + let n = try JournalCognitoOAuth.data64(key.n), e = try JournalCognitoOAuth.data64(key.e) + guard (256...1024).contains(n.count), (1...8).contains(e.count) else { throw JournalCognitoError.invalidToken } + func tlv(_ tag: UInt8, _ data: Data) -> Data { + let count = data.count + var length = Data() + if count < 128 { length.append(UInt8(count)) } + else { + var size = count, bytes: [UInt8] = [] + while size > 0 { bytes.insert(UInt8(size & 255), at: 0); size >>= 8 } + length.append(UInt8(128 + bytes.count)); length.append(contentsOf: bytes) + } + return Data([tag]) + length + data + } + func integer(_ data: Data) -> Data { + tlv(2, (data.first! >= 128 ? Data([0]) : Data()) + data) + } + let der = tlv(48, integer(n) + integer(e)) + guard let publicKey = SecKeyCreateWithData(der as CFData, + [kSecAttrKeyType: kSecAttrKeyTypeRSA, kSecAttrKeyClass: kSecAttrKeyClassPublic] as CFDictionary, nil), + SecKeyVerifySignature(publicKey, .rsaSignatureMessagePKCS1v15SHA256, + Data((parts[0] + "." + parts[1]).utf8) as CFData, + try JournalCognitoOAuth.data64(parts[2]) as CFData, nil), + let result = try JSONSerialization.jsonObject(with: JournalCognitoOAuth.data64(parts[1])) as? [String: Any] + else { throw JournalCognitoError.invalidToken } + return result + } + func validate(_ response: JournalCognitoTokenResponse, configuration: JournalCognitoConfiguration, + nonce: String?, previous: JournalCognitoTokens?, now: Date) throws -> JournalCognitoTokens { + let id = try claims(response.id_token), access = try claims(response.access_token) + func expiry(_ claims: [String: Any], use: String) throws -> Date { + guard claims["iss"] as? String == configuration.issuer, claims["token_use"] as? String == use, + let exp = claims["exp"] as? Double, exp.isFinite, exp > now.timeIntervalSince1970, + let iat = claims["iat"] as? Double, iat <= now.timeIntervalSince1970 + 60 + else { throw JournalCognitoError.invalidToken } + return Date(timeIntervalSince1970: exp) + } + let expiryID = try expiry(id, use: "id"), expiryAccess = try expiry(access, use: "access") + guard id["aud"] as? String == configuration.clientID, + access["client_id"] as? String == configuration.clientID, + let user = id["sub"] as? String, !user.isEmpty, user.utf8.count <= 512, + access["sub"] as? String == user, previous == nil || previous?.userID == user, + nonce == nil || id["nonce"] as? String == nonce, + response.token_type.lowercased() == "bearer", response.expires_in > 0, + let refresh = response.refresh_token ?? previous?.refreshToken, !refresh.isEmpty, + refresh.utf8.count <= 32768 else { throw JournalCognitoError.invalidToken } + return JournalCognitoTokens(idToken: response.id_token, accessToken: response.access_token, + refreshToken: refresh, userID: user, issuer: configuration.issuer, clientID: configuration.clientID, + expiresAt: min(expiryID, expiryAccess, now.addingTimeInterval(TimeInterval(response.expires_in)))) + } +} diff --git a/swift/JournalCognitoSession.swift b/swift/JournalCognitoSession.swift new file mode 100644 index 0000000..54ecf9e --- /dev/null +++ b/swift/JournalCognitoSession.swift @@ -0,0 +1,184 @@ +import Foundation + +@MainActor struct JournalCognitoStorage { + let load: () async throws -> JournalCognitoTokens? + let save: (JournalCognitoTokens) async throws -> Void + let clear: () async throws -> Void + var retireLegacy: () async throws -> Void = { } +} +@MainActor struct JournalCognitoBrowser { + let authenticate: (URL, String) async throws -> URL + let cancel: () -> Void +} + +/// Sole OAuth session owner. Graph/account/E2EE policy remains behind JournalPlatformServices. +@MainActor final class JournalCognitoSession: JournalAuthCapability, JournalAuthenticationAPI { + static let changed = Notification.Name("com.logseq.journal.authentication-changed") + private let configuration: JournalCognitoConfiguration + private let storage: JournalCognitoStorage + private let browser: JournalCognitoBrowser + private let transport: (URLRequest) async throws -> (Data, HTTPURLResponse) + private let now: () -> Date + private var tokens: JournalCognitoTokens? + private var loaded = false + private var generation: UInt64 = 0 + private var signInID: UUID? + private var refresh: (id: UUID, task: Task)? + private var signingOut = false + + init(configuration: JournalCognitoConfiguration, storage: JournalCognitoStorage, + browser: JournalCognitoBrowser, transport: @escaping (URLRequest) async throws -> (Data, HTTPURLResponse), + now: @escaping () -> Date = Date.init) { + self.configuration = configuration; self.storage = storage + self.browser = browser; self.transport = transport; self.now = now + } + private func check(_ version: UInt64) throws { + try Task.checkCancellation() + guard generation == version, !signingOut else { throw CancellationError() } + } + private func restore() async throws { + guard !loaded else { return } + let version = generation, value = try await storage.load() + try check(version) + if let value { + guard value.issuer == configuration.issuer, value.clientID == configuration.clientID, + !value.userID.isEmpty, !value.idToken.isEmpty, !value.accessToken.isEmpty, !value.refreshToken.isEmpty + else { throw JournalCognitoError.invalidToken } + } + // A concurrent restore may have completed a refresh or sign-in already. + if !loaded { tokens = value; loaded = true } + } + func currentUserID() async throws -> String? { + try await restore() + guard tokens != nil else { return nil } + _ = try await freshIDToken() + return tokens?.userID + } + func freshIDToken() async throws -> String { + try await restore() + guard !signingOut, let current = tokens else { throw JournalPlatformServices.Failure.authenticationRequired } + guard current.expiresAt.timeIntervalSince(now()) <= 60 else { return current.idToken } + let version = generation + let operation: (id: UUID, task: Task) + if let refresh { operation = refresh } + else { + let id = UUID() + let task = Task { [self] in + do { + let response = try await requestTokens(["grant_type": "refresh_token", "client_id": configuration.clientID, + "refresh_token": current.refreshToken]) + try check(version) + let updated = try await validate(response, nonce: nil, previous: current) + try check(version) + try await storage.save(updated) + try check(version) + tokens = updated + return updated + } catch JournalCognitoError.invalidGrant { + try check(version) + tokens = nil; loaded = true + try await storage.clear() + try check(version) + NotificationCenter.default.post(name: Self.changed, object: self) + throw JournalPlatformServices.Failure.authenticationRequired + } + } + operation = (id, task); refresh = operation + } + defer { if refresh?.id == operation.id { refresh = nil } } + let result = try await operation.task.value + try check(version) + return result.idToken + } + func signIn() async throws { + guard signInID == nil, !signingOut else { throw JournalCognitoError.unavailable } + let version = generation, id = UUID(), started = now() + signInID = id + defer { if signInID == id { signInID = nil } } + try await restore() + try check(version) + guard signInID == id else { throw CancellationError() } + let state = try JournalCognitoOAuth.random(), verifier = try JournalCognitoOAuth.random(), nonce = try JournalCognitoOAuth.random() + let url = try JournalCognitoOAuth.authorization(configuration, state: state, verifier: verifier, nonce: nonce) + func admitted() throws { + try check(version) + guard signInID == id else { throw CancellationError() } + guard now().timeIntervalSince(started) <= 300 else { throw JournalCognitoError.expired } + } + let callback = try await browser.authenticate(url, "logseqjournal") + try admitted() + let code = try JournalCognitoOAuth.code(callback, configuration: configuration, state: state) + let response = try await requestTokens(["grant_type": "authorization_code", "client_id": configuration.clientID, + "redirect_uri": configuration.redirectURI, "code": code, "code_verifier": verifier]) + try admitted() + let updated = try await validate(response, nonce: nonce, previous: nil) + try admitted() + // Retire any refresh of the preceding account before persisting the replacement. + let previous = tokens + generation &+= 1; refresh?.task.cancel(); refresh = nil + let savingVersion = generation + try await storage.save(updated) + if generation == savingVersion && (signInID != id || Task.isCancelled) { + // Storage is serialized. Undo only this cancelled write; never overwrite a + // newer sign-in or explicit sign-out that has advanced the generation. + if let previous { try await storage.save(previous) } + else { try await storage.clear() } + throw CancellationError() + } + try check(savingVersion) + guard signInID == id else { throw CancellationError() } + tokens = updated; loaded = true + signInID = nil + try? await storage.retireLegacy() + try check(savingVersion) + NotificationCenter.default.post(name: Self.changed, object: self) + } + func cancelSignIn() { + guard signInID != nil else { return } + signInID = nil; browser.cancel() + } + func signOut() async throws { + guard !signingOut else { throw JournalCognitoError.unavailable } + generation &+= 1; signingOut = true + cancelSignIn(); refresh?.task.cancel(); refresh = nil + defer { signingOut = false } + var previous = tokens + if previous == nil { previous = try? await storage.load() } + tokens = nil; loaded = true + try await storage.clear() + try await storage.retireLegacy() + NotificationCenter.default.post(name: Self.changed, object: self) + if let previous { + // Local cleanup succeeds offline. Revocation is bounded and best effort. + _ = try? await transport(request(path: "/oauth2/revoke", values: ["client_id": configuration.clientID, + "token": previous.refreshToken])) + } + } + private func request(path: String, values: [String: String]) -> URLRequest { + var request = URLRequest(url: URL(string: "https://\(configuration.domain)\(path)")!) + request.httpMethod = "POST"; request.timeoutInterval = 15 + request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type") + request.httpBody = JournalCognitoOAuth.form(values) + return request + } + private func requestTokens(_ values: [String: String]) async throws -> JournalCognitoTokenResponse { + let request = request(path: "/oauth2/token", values: values) + let (data, response) = try await transport(request) + guard response.url == request.url, data.count <= 131072 else { throw JournalCognitoError.request } + guard response.statusCode == 200 else { + let error = (try? JSONSerialization.jsonObject(with: data) as? [String: Any])?["error"] as? String + if response.statusCode == 400, error == "invalid_grant" { throw JournalCognitoError.invalidGrant } + throw JournalCognitoError.request + } + do { return try JSONDecoder().decode(JournalCognitoTokenResponse.self, from: data) } + catch { throw JournalCognitoError.invalidToken } + } + private func validate(_ response: JournalCognitoTokenResponse, nonce: String?, previous: JournalCognitoTokens?) async throws -> JournalCognitoTokens { + var request = URLRequest(url: URL(string: configuration.issuer + "/.well-known/jwks.json")!) + request.timeoutInterval = 15 + let (data, http) = try await transport(request) + guard http.statusCode == 200, http.url == request.url else { throw JournalCognitoError.request } + return try JournalCognitoTokenValidator(jwks: data).validate(response, configuration: configuration, + nonce: nonce, previous: previous, now: now()) + } +} diff --git a/swift/JournalLUIEvents.swift b/swift/JournalLUIEvents.swift index 6dc7bfd..256700d 100644 --- a/swift/JournalLUIEvents.swift +++ b/swift/JournalLUIEvents.swift @@ -10,6 +10,30 @@ private func luiOCamlAppear(_ node: Int64) -> Int32 private func luiOCamlPress(_ node: Int64) -> Int32 @_silgen_name("lui_ocaml_long_press") private func luiOCamlLongPress(_ node: Int64) -> Int32 +@_silgen_name("lui_ocaml_press_detail") +private func luiOCamlPressDetail( + _ node: Int64, _ x: Double, _ y: Double, _ modifiers: Int32, _ button: Int32, + _ targetClass: UnsafePointer? +) -> Int32 +@_silgen_name("lui_ocaml_pointer_down") +private func luiOCamlPointerDown( + _ node: Int64, _ x: Double, _ y: Double, _ modifiers: Int32, _ button: Int32, + _ targetClass: UnsafePointer? +) -> Int32 +@_silgen_name("lui_ocaml_pointer_up") +private func luiOCamlPointerUp( + _ node: Int64, _ x: Double, _ y: Double, _ modifiers: Int32, _ button: Int32, + _ targetClass: UnsafePointer? +) -> Int32 +@_silgen_name("lui_ocaml_context_menu_press") +private func luiOCamlContextMenuPress( + _ node: Int64, _ x: Double, _ y: Double, _ modifiers: Int32, _ button: Int32, + _ targetClass: UnsafePointer? +) -> Int32 +@_silgen_name("lui_ocaml_pointer_enter") +private func luiOCamlPointerEnter(_ node: Int64) -> Int32 +@_silgen_name("lui_ocaml_pointer_leave") +private func luiOCamlPointerLeave(_ node: Int64) -> Int32 @_silgen_name("lui_ocaml_text_changed") private func luiOCamlTextChanged(_ node: Int64, _ text: UnsafePointer?) -> Int32 @_silgen_name("lui_ocaml_submit") @@ -43,6 +67,24 @@ private func journalOCamlExtensionEvent( case .appear(let node): return luiOCamlAppear(Int64(node)) case .press(let node): return luiOCamlPress(Int64(node)) case .longPress(let node): return luiOCamlLongPress(Int64(node)) + case .pressDetail(let node, let x, let y, let modifiers, let button, let targetClass): + return targetClass.withCString { + luiOCamlPressDetail(Int64(node), x, y, Int32(modifiers), Int32(button), $0) + } + case .pointerDown(let node, let x, let y, let modifiers, let button, let targetClass): + return targetClass.withCString { + luiOCamlPointerDown(Int64(node), x, y, Int32(modifiers), Int32(button), $0) + } + case .pointerUp(let node, let x, let y, let modifiers, let button, let targetClass): + return targetClass.withCString { + luiOCamlPointerUp(Int64(node), x, y, Int32(modifiers), Int32(button), $0) + } + case .contextMenuPress(let node, let x, let y, let modifiers, let button, let targetClass): + return targetClass.withCString { + luiOCamlContextMenuPress(Int64(node), x, y, Int32(modifiers), Int32(button), $0) + } + case .pointerEnter(let node): return luiOCamlPointerEnter(Int64(node)) + case .pointerLeave(let node): return luiOCamlPointerLeave(Int64(node)) case .textChanged(let node, let text): return text.withCString { luiOCamlTextChanged(Int64(node), $0) } case .submit(let node): return luiOCamlSubmit(Int64(node)) diff --git a/swift/JournalNativeServices.swift b/swift/JournalNativeServices.swift index b358fd5..a6989b7 100644 --- a/swift/JournalNativeServices.swift +++ b/swift/JournalNativeServices.swift @@ -6,7 +6,7 @@ import Foundation static func makePlatformServices() -> JournalPlatformServices { let storage = JournalNativeAccountStorage() return JournalPlatformServices( - auth: JournalAmplifySession(), + auth: JournalCognitoNative.shared, account: JournalAccountStore( load: { try await storage.load() }, save: { try await storage.save($0) }, diff --git a/swift/Package.resolved b/swift/Package.resolved deleted file mode 100644 index 47e1460..0000000 --- a/swift/Package.resolved +++ /dev/null @@ -1,276 +0,0 @@ -{ - "originHash" : "9049adbec490a2f21999f1cbe54ea98c658c6d28da055f12332c856b60edbfef", - "pins" : [ - { - "identity" : "amplify-swift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/aws-amplify/amplify-swift.git", - "state" : { - "revision" : "ce8d9e69df07e80c57e2ca68c01360ea6636abf5", - "version" : "2.61.0" - } - }, - { - "identity" : "amplify-swift-utils-notifications", - "kind" : "remoteSourceControl", - "location" : "https://github.com/aws-amplify/amplify-swift-utils-notifications.git", - "state" : { - "revision" : "959eec669ba97c7d923b963c3e66ca8a0b2737f6", - "version" : "1.1.1" - } - }, - { - "identity" : "async-http-client", - "kind" : "remoteSourceControl", - "location" : "https://github.com/swift-server/async-http-client.git", - "state" : { - "revision" : "f95c908967e98c68c5ce3fd61a7974e7e869e303", - "version" : "1.36.1" - } - }, - { - "identity" : "aws-crt-swift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/awslabs/aws-crt-swift", - "state" : { - "revision" : "d1678252cb3da2d34f70db43e402927360df4d8c", - "version" : "0.64.1" - } - }, - { - "identity" : "aws-sdk-swift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/awslabs/aws-sdk-swift", - "state" : { - "revision" : "61b8c1968aeae4d42226e3b452ab864fd0b96523", - "version" : "1.7.60" - } - }, - { - "identity" : "smithy-swift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/smithy-lang/smithy-swift", - "state" : { - "revision" : "31336dfea2e448523630ea1fe4d83ddc1156de3c", - "version" : "0.242.0" - } - }, - { - "identity" : "sqlite.swift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/stephencelis/SQLite.swift.git", - "state" : { - "revision" : "392dd6058624d9f6c5b4c769d165ddd8c7293394", - "version" : "0.15.4" - } - }, - { - "identity" : "swift-algorithms", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-algorithms.git", - "state" : { - "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", - "version" : "1.2.1" - } - }, - { - "identity" : "swift-argument-parser", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-argument-parser.git", - "state" : { - "revision" : "6a52f3251125d74daf04fcbd5e6f08a75d074382", - "version" : "1.8.2" - } - }, - { - "identity" : "swift-asn1", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-asn1.git", - "state" : { - "revision" : "3b6410f7dee09eb33cdd26260c5fd47fda19b0e2", - "version" : "1.7.3" - } - }, - { - "identity" : "swift-async-algorithms", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-async-algorithms.git", - "state" : { - "revision" : "3da39bbc4e687d4192af7c9cf4eab805745a0b9c", - "version" : "1.1.5" - } - }, - { - "identity" : "swift-atomics", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-atomics.git", - "state" : { - "revision" : "0442cb5a3f98ab802acb777929fdb446bda11a34", - "version" : "1.3.1" - } - }, - { - "identity" : "swift-certificates", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-certificates.git", - "state" : { - "revision" : "ff86b924ead66f853b8baf91f3c41926a8f36177", - "version" : "1.21.0" - } - }, - { - "identity" : "swift-collections", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-collections", - "state" : { - "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", - "version" : "1.6.0" - } - }, - { - "identity" : "swift-configuration", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-configuration.git", - "state" : { - "revision" : "3533f65d3e36dcdffc91ce34ef4d3c9c1887fd4b", - "version" : "1.2.1" - } - }, - { - "identity" : "swift-crypto", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-crypto.git", - "state" : { - "revision" : "a9d1d5ab8951ada40cafff8c8e2b551dfde4f390", - "version" : "5.0.0" - } - }, - { - "identity" : "swift-distributed-tracing", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-distributed-tracing.git", - "state" : { - "revision" : "cc504a45f6ce73ce6067837d7ac19fa67b229a56", - "version" : "1.5.0" - } - }, - { - "identity" : "swift-http-structured-headers", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-http-structured-headers.git", - "state" : { - "revision" : "933538faa42c432d385f02e07df0ace7c5ecfc47", - "version" : "1.7.0" - } - }, - { - "identity" : "swift-http-types", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-http-types.git", - "state" : { - "revision" : "bff4b6903cdc99dda49649dd52f46c11cfd3ed50", - "version" : "1.8.0" - } - }, - { - "identity" : "swift-log", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-log.git", - "state" : { - "revision" : "9c6fb14227f55d8f711ce3847dc2f419fb0ecacb", - "version" : "1.15.1" - } - }, - { - "identity" : "swift-nio", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-nio.git", - "state" : { - "revision" : "21de5f08c1a166a6dd293d0e587ad977bf8dac5d", - "version" : "2.103.0" - } - }, - { - "identity" : "swift-nio-extras", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-nio-extras.git", - "state" : { - "revision" : "41449336c8ecfadac6b4b5be75f9c3c306e61ced", - "version" : "1.35.1" - } - }, - { - "identity" : "swift-nio-http2", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-nio-http2.git", - "state" : { - "revision" : "0f3e54e29c944c2e835ad52159da7d9e1c94ac69", - "version" : "1.46.0" - } - }, - { - "identity" : "swift-nio-ssl", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-nio-ssl.git", - "state" : { - "revision" : "322f3c2a4a21df31c84ca416bf65ee5e9059e440", - "version" : "2.37.5" - } - }, - { - "identity" : "swift-nio-transport-services", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-nio-transport-services.git", - "state" : { - "revision" : "67787bb645a5e67d2edcdfbe48a216cc549222d5", - "version" : "1.28.0" - } - }, - { - "identity" : "swift-numerics", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-numerics.git", - "state" : { - "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", - "version" : "1.1.1" - } - }, - { - "identity" : "swift-service-context", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-service-context.git", - "state" : { - "revision" : "d0997351b0c7779017f88e7a93bc30a1878d7f29", - "version" : "1.3.0" - } - }, - { - "identity" : "swift-service-lifecycle", - "kind" : "remoteSourceControl", - "location" : "https://github.com/swift-server/swift-service-lifecycle", - "state" : { - "revision" : "7f9326b0326ff86e3646295ea6e891f68c471c5e", - "version" : "2.12.0" - } - }, - { - "identity" : "swift-system", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-system", - "state" : { - "revision" : "869129b7bf4ecc57b97d0193ad29690ca2134750", - "version" : "1.8.1" - } - }, - { - "identity" : "swift-toolchain-sqlite", - "kind" : "remoteSourceControl", - "location" : "https://github.com/swiftlang/swift-toolchain-sqlite", - "state" : { - "revision" : "b626d3002773b1a1304166643e7f118f724b2132", - "version" : "1.0.4" - } - } - ], - "version" : 3 -} diff --git a/swift/Package.swift b/swift/Package.swift index 540f710..e9287d2 100644 --- a/swift/Package.swift +++ b/swift/Package.swift @@ -25,18 +25,12 @@ let package = Package( platforms: [.iOS("26.0"), .macOS("26.0")], dependencies: [ .package(path: luiPackagePath), - .package( - url: "https://github.com/aws-amplify/amplify-swift.git", - exact: "2.61.0" - ), ], targets: [ .executableTarget( name: "JournalApp", dependencies: [ .product(name: "LUIAppleBackendStatic", package: "apple"), - .product(name: "Amplify", package: "amplify-swift"), - .product(name: "AWSCognitoAuthPlugin", package: "amplify-swift"), ], path: ".", exclude: ["Package.swift"], diff --git a/test/application_view_test.ml b/test/application_view_test.ml index ce46e6e..098bc7a 100644 --- a/test/application_view_test.ml +++ b/test/application_view_test.ml @@ -1,4 +1,57 @@ module Service = Logseq_db_worker_lui.Logseq_db_worker_lui_service +module Wire_nodes = Set.Make (Int) + +let track_wire_teardown ?(parents = Hashtbl.create 64) on_drop = + let children = Hashtbl.create 64 in + let children_of node = + Option.value (Hashtbl.find_opt children node) ~default:Wire_nodes.empty + in + let unlink node = + match Hashtbl.find_opt parents node with + | None -> () + | Some parent -> + Hashtbl.replace children parent (Wire_nodes.remove node (children_of parent)); + Hashtbl.remove parents node + in + let drop node = + unlink node; + Wire_nodes.iter (Hashtbl.remove parents) (children_of node); + Hashtbl.remove children node; + on_drop node + in + let rec detach node = + Wire_nodes.iter detach (children_of node); + drop node + in + function + | Lui_protocol.InsertChild (parent, child, _) | MoveChild (parent, child, _) -> + unlink child; + Hashtbl.replace parents child parent; + Hashtbl.replace children parent (Wire_nodes.add child (children_of parent)) + | RemoveChild (parent, child) -> + if Hashtbl.find_opt parents child = Some parent then unlink child + | DropNode node -> drop node + | DetachSubtree node -> detach node + | _ -> () +;; + +let track_json_wire_teardown ?parents on_drop = + let track = track_wire_teardown ?parents on_drop in + fun op -> + let open Yojson.Safe.Util in + let id () = member "id" op |> to_int in + let parent () = member "parent" op |> to_int in + let child () = member "child" op |> to_int in + match member "op" op |> to_string with + | "drop-node" | "drop-extension" -> track (Lui_protocol.DropNode (id ())) + | "detach-subtree" -> track (Lui_protocol.DetachSubtree (id ())) + | "insert-child" -> + track (Lui_protocol.InsertChild (parent (), child (), member "index" op |> to_int)) + | "move-child" -> + track (Lui_protocol.MoveChild (parent (), child (), member "index" op |> to_int)) + | "remove-child" -> track (Lui_protocol.RemoveChild (parent (), child ())) + | _ -> () +;; let check_pairs label expected actual = Alcotest.(check (list (pair string string))) label expected actual @@ -338,6 +391,7 @@ let run_favorites_native_visibility ?(check_chrome = false) ?(check_error_control = false) ?(check_ios_capture = false) + ?(check_capture_imports = false) ?(on_initialized = fun () -> ()) ?media_rows ?(shared_media = false) @@ -369,8 +423,17 @@ let run_favorites_native_visibility let acquire_release = Atomic.make false in let released_files = Atomic.make 0 in let media_checksum = Atomic.make 'a' in + let media_metadata_changed = Atomic.make false in let detail_fixture = Atomic.make false in let released_demands = Atomic.make 0 in + let capture_root : P.v2_block_tree option Atomic.t = Atomic.make None in + let capture_page : G.page option Atomic.t = Atomic.make None in + let imports_entered = Atomic.make [] in + let imports_finished = Atomic.make 0 in + let import_release = Atomic.make false in + let import_probe = Atomic.make false in + let import_probed = Atomic.make false in + let staged_paths = ref [] in let scope : Service.asset_scope = { account = { managed_sync_origin = Uri.of_string "https://example.invalid" @@ -383,30 +446,6 @@ let run_favorites_native_visibility ; graph_generation = 1 } in - let items = - List.mapi - (fun n root -> - { P.membership_uuid = uuid (1000 + n) - ; membership_order = Printf.sprintf "%03d" n - ; membership_revision = "membership-1" - ; target = - (if n mod 2 = 0 - then - P.V2_favorite_page - { uuid = root - ; title = Printf.sprintf "Fixture %d" n - ; revision = "page-1" - } - else - P.V2_favorite_block - { uuid = root - ; title = Printf.sprintf "Fixture %d" n - ; task_status = None - ; revision = "block-1" - }) - }) - roots - in let page : G.page = { uuid = uuid 10000 ; name = "20260901" @@ -440,6 +479,69 @@ let run_favorites_native_visibility ; tag_titles = [] } in + let favorites_page = + { page with + uuid = uuid 40000 + ; name = "$$$favorites" + ; title = "Favorites" + ; kind = Hidden_page + } + in + let favorite_target_uuid n = + if n mod 2 = 0 then uuid (30000 + n) else List.nth roots n + in + let favorite_page n = + { page with + uuid = favorite_target_uuid n + ; name = Printf.sprintf "fixture-%d" n + ; title = Printf.sprintf "Fixture %d" n + ; kind = Ordinary_page + } + in + let favorite_label n = + if n mod 2 = 0 then (favorite_page n).title else (timeline_block n).block.title + in + let favorite_items () = + List.init 65 (fun n -> + { P.membership_uuid = uuid (1000 + n) + ; membership_order = Printf.sprintf "%03d" n + ; membership_revision = "membership-1" + ; target = + (if n mod 2 = 0 + then + P.V2_favorite_page + { uuid = favorite_target_uuid n + ; title = favorite_label n + ; revision = "page-1" + } + else + P.V2_favorite_block + { uuid = favorite_target_uuid n + ; title = favorite_label n + ; task_status = None + ; revision = "root" + }) + }) + in + let favorite_membership n = + let value = timeline_block n in + { P.value = + { value with + block = + { value.block with + uuid = uuid (1000 + n) + ; title = "" + ; parent = favorites_page.uuid + ; page = favorites_page.uuid + ; refs = [] + ; properties = [] + } + ; rendered_page_title = favorites_page.title + } + ; revision = "membership-1" + } + in + let changes_acknowledged = Atomic.make false in let manager : Service.state = { snapshot = { sync_phase = Current @@ -469,7 +571,8 @@ let run_favorites_native_visibility W.Service.create ~push_topic_count:6 ~merge_push:Service.coalesce_push - ~concurrency:Serial + ~concurrency: + (if check_capture_imports then Concurrent { max_in_flight = 2 } else Serial) ~init:(fun context _ -> worker_context := Some context; W.Session_context.emit @@ -480,6 +583,7 @@ let run_favorites_native_visibility ~handle:(fun context () request -> match request with | Service.Get_graph_state -> + if Atomic.get import_probe then Atomic.set import_probed true; if Atomic.exchange publish_block false then ( Atomic.set updated_block true; @@ -548,6 +652,44 @@ let run_favorites_native_visibility Ok Service.Client_command_completed | Client_command _ | Asset_command _ -> Ok Service.Client_command_completed | Acquire_asset_file _ | Acquire_imported_file _ -> Ok (Service.Asset_file None) + | Import_asset { graph_generation; source } when check_capture_imports -> + Atomic.set imports_entered (Atomic.get imports_entered @ [ source.title ]); + if source.title = "First attachment" + then ( + while not (Atomic.get import_release) do + Eio.Time.Mono.sleep (W.Request_context.clock context) 0.001 + done; + Atomic.incr imports_finished; + Ok (Service.Asset_imported (Error "Fixture first attachment failed"))) + else + let module A = Logseq_db_types.Asset_descriptor in + let asset = + A.create + ~uuid:source.asset + ~source: + (Managed + (Some + (A.version + ~checksum:(String.make 64 'a') + ~file_type:source.file_type + |> Result.get_ok))) + ~current_checksum:None + ~size:None + ~dimensions:None + |> Result.get_ok + in + Atomic.incr imports_finished; + Ok + (Service.Asset_imported + (Ok + { operation = source.operation + ; graph_generation + ; scope + ; target = source.target + ; asset + ; file_type = source.file_type + ; preview = None + })) | Import_asset _ -> Error "unexpected import" | Graph_request request -> let outcome = @@ -600,19 +742,125 @@ let run_favorites_native_visibility [ { id = "covered-update" ; predecessor = "p" ; successor = "p2" - ; block_uuids = [ uuid 2 ] + ; block_uuids = + (if Atomic.get media_metadata_changed + then [ uuid 20000 ] + else [ uuid 2 ]) ; page_uuids = [] ; structure_interests = [] } ] } | V2_ack_changes _ -> + Atomic.set changes_acknowledged true; V2_changes_acknowledged { generation = "g"; through = "p2" } | V2_get_block { block; _ } -> - let n = List.find_index (G.Uuid.equal block) roots |> Option.get in V2_block_outcome - (V2_present_block { value = timeline_block n; revision = "root" }) - | V2_get_page _ -> V2_page_outcome (V2_present_page { page; revision = "p" }) + (match List.find_index (G.Uuid.equal block) roots with + | Some n -> + V2_present_block { value = timeline_block n; revision = "root" } + | None -> V2_missing_block { uuid = block; revision = "p" }) + | V2_get_block_summary { block; _ } when check_capture_imports -> + let root = Atomic.get capture_root |> Option.get in + let page = Atomic.get capture_page |> Option.get in + Alcotest.(check string) + "capture summary queries committed UUID" + (G.Uuid.to_string root.uuid) + (G.Uuid.to_string block); + let value = timeline_block 0 in + V2_block_summary_outcome + { lookup = + V2_present_block + { value = + { value with + block = + { value.block with + uuid = root.uuid + ; title = root.title + ; parent = page.uuid + ; page = page.uuid + } + ; rendered_page_title = page.title + } + ; revision = "captured" + } + ; page = Some (V2_present_page { page; revision = "p" }) + ; items = [] + ; next_cursor = None + ; scope_revision = "children" + ; generation = "g" + ; projection_revision = "p2" + } + | V2_get_page { page = requested; _ } -> + let known = + if G.Uuid.equal requested page.uuid + then Some page + else if G.Uuid.equal requested favorites_page.uuid + then Some favorites_page + else + List.find_map + (fun n -> + if n mod 2 = 0 && G.Uuid.equal requested (favorite_target_uuid n) + then Some (favorite_page n) + else None) + (List.init 65 Fun.id) + in + let capture_day = + if not check_capture_imports + then None + else ( + try + Scanf.sscanf + (G.Uuid.to_string requested) + "00000001-%4d-%4d-0000-000000000000%!" + (fun year month_day -> + let month = month_day / 100 + and day = month_day mod 100 in + let leap = + year mod 4 = 0 && (year mod 100 <> 0 || year mod 400 = 0) + in + let maximum_day = + match month with + | 2 -> if leap then 29 else 28 + | 4 | 6 | 9 | 11 -> 30 + | 1 | 3 | 5 | 7 | 8 | 10 | 12 -> 31 + | _ -> 0 + in + if + year > 0 + && day >= 1 + && day <= maximum_day + && G.Uuid.to_string requested + = Printf.sprintf + "00000001-%04d-%04d-0000-000000000000" + year + month_day + then Some ((year * 10000) + month_day) + else None) + with + | Scanf.Scan_failure _ | End_of_file | Failure _ -> None) + in + let known = + match known, capture_day with + | Some _, _ -> known + | None, Some journal_day -> + let name = Printf.sprintf "%08d" journal_day in + let captured = + { page with + uuid = requested + ; name + ; title = name + ; kind = Journal_page { journal_day } + } + in + Atomic.set capture_page (Some captured); + Some captured + | None, None -> None + in + V2_page_outcome + (match known with + | Some page -> V2_present_page { page; revision = "p" } + | None -> V2_missing_page { uuid = requested; revision = "p" }) | V2_get_children { parent; _ } when Atomic.get detail_fixture && G.Uuid.equal parent (uuid 1) -> let value = timeline_block 1 in @@ -632,16 +880,33 @@ let run_favorites_native_visibility { parent ; revision_scope = V2_children_revision parent ; scope_revision = "children" - ; items = [] + ; items = + (if G.Uuid.equal parent favorites_page.uuid + then List.init 65 favorite_membership + else []) ; next_cursor = None } - | V2_list_favorites _ -> + | V2_list_favorites { limit; cursor } -> + let offset = + match cursor with + | None -> 0 + | Some cursor -> int_of_string (G.Cursor.to_string cursor) + in + let items = + favorite_items () + |> List.filteri (fun n _ -> n >= offset && n < offset + limit) + in + let through = offset + List.length items in V2_favorites_outcome - { favorites_page = None + { favorites_page = Some favorites_page.uuid ; generation = "g" ; projection_revision = "p" ; items - ; next_cursor = None + ; next_cursor = + (if through < 65 + then + Some (G.Cursor.of_string (string_of_int through) |> Result.get_ok) + else None) } | V2_list_assets { roots; _ } -> Atomic.set queried (List.rev_append roots (Atomic.get queried)); @@ -680,6 +945,21 @@ let run_favorites_native_visibility else []) ; next_cursor = None } + | V2_insert_blocks { mutation_id; parent; roots = [ root ]; _ } + when check_capture_imports -> + let page = Atomic.get capture_page |> Option.get in + Alcotest.(check string) + "capture mutation targets its requested journal page" + (G.Uuid.to_string page.uuid) + (G.Uuid.to_string parent); + Atomic.set capture_root (Some root); + V2_mutation_committed + { mutation_id + ; status = V2_applied + ; generation = "g" + ; before_projection_revision = "p" + ; after_projection_revision = "p2" + } | _ -> V2_failed { code = "unsupported"; message = "Unused fixture command" } in Ok @@ -706,6 +986,7 @@ let run_favorites_native_visibility in let props = Hashtbl.create 512 and parents = Hashtbl.create 512 in + let track_teardown = track_json_wire_teardown ~parents (Hashtbl.remove props) in let consume encoded = if encoded <> "" then @@ -714,6 +995,7 @@ let run_favorites_native_visibility |> member "ops" |> to_list |> List.iter (fun op -> + track_teardown op; match op |> member "op" |> to_string with | "create-node" -> Hashtbl.replace @@ -753,15 +1035,6 @@ let run_favorites_native_visibility props id ((key, member "value" op) :: List.remove_assoc key previous) - | "drop-node" | "drop-extension" -> - let id = op |> member "id" |> to_int in - Hashtbl.remove props id; - Hashtbl.remove parents id - | "insert-child" | "move-child" -> - Hashtbl.replace - parents - (op |> member "child" |> to_int) - (op |> member "parent" |> to_int) | _ -> ()) in let find key value = @@ -790,6 +1063,41 @@ let run_favorites_native_visibility done; Alcotest.(check bool) label true (predicate ()) in + let load_all_favorites label = + wait label (fun () -> Option.is_some (find "text" "Fixture 48")); + if Option.is_none (find "text" "Fixture 64") + then ( + let rec list_ancestor id = + if + List.assoc_opt + "_extension" + (Option.value (Hashtbl.find_opt props id) ~default:[]) + = Some (`String "journal-list") + then id + else list_ancestor (Hashtbl.find parents id) + in + let list_node = list_ancestor (Option.get (find "text" "Fixture 48")) in + hooks.extension_event + list_node + "event" + {|{"id":1,"payload":"{\"type\":\"visible_range\",\"first\":38,\"last\":50}"}|} + |> consume); + wait "second Favorite page loaded" (fun () -> + Option.is_some (find "text" "Fixture 64")) + in + let settle_worker () = + let idle_rounds = ref 0 in + wait "background hydration completes" (fun () -> + let client = Option.get !client in + let metrics = W.Private.metrics (W.Private.pack_client client) in + if + metrics.queued_requests = 0 + && metrics.active_request_fibers = 0 + && W.For_testing.pending_output_count client = 0 + then incr idle_rounds + else idle_rounds := 0; + !idle_rounds >= 12) + in let startup = Logseq_db_worker.Config.create ~application_support_directory:"/tmp/journal-favorites-synthetic" @@ -804,14 +1112,16 @@ let run_favorites_native_visibility in Fun.protect ~finally:(fun () -> + Atomic.set import_release true; Atomic.set acquire_release true; ignore (hooks.dispose ()); Option.iter Logseq_db_worker_lui.Journal_worker_runtime.stop !client; + List.iter (fun path -> if Sys.file_exists path then Sys.remove path) !staged_paths; check_fixture_worker_idle ()) (fun () -> hooks.init 2 2 startup |> consume; on_initialized (); - if check_ios_capture + if check_ios_capture || check_capture_imports then ( let snapshot = { Journal_environment.fallback with platform = "ios" } in let payload = @@ -1006,8 +1316,7 @@ let run_favorites_native_visibility !clipboard_requests <> []); dispatch (Lui_protocol.Press (Option.get (find "accessibility-label" "Favorites"))); - wait "copy Favorites loaded" (fun () -> - Option.is_some (find "text" "Fixture 64")); + load_all_favorites "copy Favorites loaded"; let list = Option.get (find "_extension" "journal-list") in check_copy_text list (G.Uuid.to_string (uuid 1001)) "Timeline fixture 1"; clipboard_requests := []; @@ -1578,11 +1887,23 @@ let run_favorites_native_visibility publish (Service.Asset.Failed { failure = Network; attempts = 1; retry_scheduled = false }); - wait "invalidated file closes preview" (fun () -> previews () = []); + settle (); + Alcotest.(check bool) + "late Failed retains successful preview URL" + true + (List.mem preview (previews ()) && preview_has_path preview); + Alcotest.(check int) + "late Failed releases no successful file" + 0 + (Atomic.get released_files); Alcotest.(check int) - "invalidated preview leaves no URL" + "late Failed releases no preview demand" 0 - (List.length (files ()))) + (Atomic.get released_demands); + Alcotest.(check int) + "late Failed cannot reacquire successful file" + acquired + (Atomic.get acquire_entered)) else ( native_range 2 (count + 1); settle (); @@ -1611,15 +1932,39 @@ let run_favorites_native_visibility (List.for_all preview_has_path (previews ()))); if scenario = `Preview_replaced then ( + let reads = List.length (Atomic.get queried) in + Atomic.set changes_acknowledged false; Atomic.set media_checksum 'b'; + Atomic.set media_metadata_changed true; W.Session_context.emit (Option.get !worker_context) ~topic:Service.invalidation_topic (Service.Graph_push (P.V2_changes_available { api_version = 2; generation = "g"; through = "p2" })); - wait "descriptor replacement clears old preview" (fun () -> - files () = [])); + wait "metadata changes acknowledged" (fun () -> + Atomic.get changes_acknowledged); + settle (); + Alcotest.(check bool) + "same UUID metadata retains preview URL" + true + (List.mem preview (previews ()) && preview_has_path preview); + Alcotest.(check int) + "metadata window does not read successful File again" + reads + (List.length (Atomic.get queried)); + Alcotest.(check int) + "metadata window releases no file" + 0 + (Atomic.get released_files); + Alcotest.(check int) + "metadata window releases no demand" + 0 + (Atomic.get released_demands); + Alcotest.(check int) + "metadata window cannot reacquire successful file" + acquired + (Atomic.get acquire_entered)); if scenario = `Preview_graph then ( Atomic.set graph_generation 2; @@ -1905,6 +2250,104 @@ let run_favorites_native_visibility acquired (Atomic.get acquire_entered)) else ( + if check_capture_imports + then ( + (* Application owns the asynchronous import queue; its pure navigation + reducer cannot observe Worker handler overlap or file release. + Two Worker lanes make accidental parallel submission observable. *) + dispatch + (Lui_protocol.Press (Option.get (find "accessibility-label" "Capture"))); + dispatch (Lui_protocol.Press (Option.get (find "accessibility-label" "文件"))); + let adapter = + Hashtbl.fold + (fun id values found -> + match + List.assoc_opt "_extension" values, List.assoc_opt "payload" values + with + | Some (`String "journal-asset-import"), Some (`String payload) -> + let request = + Yojson.Safe.from_string payload |> Yojson.Safe.Util.member "request" + in + if + Yojson.Safe.Util.(member "id" request |> to_int) > 0 + && Yojson.Safe.Util.member "staged" request = `Bool true + then Some id + else found + | _ -> found) + props + None + |> Option.get + in + let request = + List.assoc "payload" (Hashtbl.find props adapter) + |> Yojson.Safe.Util.to_string + |> Yojson.Safe.from_string + |> Yojson.Safe.Util.member "request" + in + let item index title = + let path = Filename.temp_file "journal-import-" ".txt" in + staged_paths := !staged_paths @ [ path ]; + `Assoc + [ "operation", `String (G.Uuid.to_string (uuid (30000 + index))) + ; "asset", `String (G.Uuid.to_string (uuid (31000 + index))) + ; "localMutation", `String (G.Uuid.to_string (uuid (32000 + index))) + ; "metadataMutation", `String (G.Uuid.to_string (uuid (33000 + index))) + ; "path", `String path + ; "title", `String title + ; "type", `String "txt" + ; "sourceIdentity", `String title + ] + in + let batch = + `Assoc + [ "action", `String "picked-batch" + ; "request", request + ; "items", `List [ item 1 "First attachment"; item 2 "Second attachment" ] + ] + in + hooks.extension_event + adapter + "event" + (Yojson.Safe.to_string + (`Assoc [ "id", `Int 1; "payload", `String (Yojson.Safe.to_string batch) ])) + |> consume; + Alcotest.(check bool) + "picker batch mounts pending attachment" + true + (Option.is_some + (find + "accessibility-identifier" + ("composer-attachment:" ^ G.Uuid.to_string (uuid 30001)))); + dispatch (Lui_protocol.Press (Option.get (find "accessibility-label" "Send"))); + wait "first attachment enters actual Worker handler" (fun () -> + Atomic.get imports_entered <> []); + Atomic.set import_probe true; + ignore (W.send (Option.get !client) Service.Get_graph_state); + wait "second Worker lane reaches ordered request barrier" (fun () -> + Atomic.get import_probed); + Alcotest.(check (list string)) + "second import waits for first terminal response" + [ "First attachment" ] + (Atomic.get imports_entered); + Alcotest.(check bool) + "in-flight and queued staging files remain owned" + true + (List.for_all Sys.file_exists !staged_paths); + Atomic.set import_release true; + wait "second attachment continues after first failure" (fun () -> + Atomic.get imports_finished = 2); + wait "terminal import responses release both staging files" (fun () -> + List.for_all (fun path -> not (Sys.file_exists path)) !staged_paths); + Alcotest.(check (list string)) + "batch preserves picker order" + [ "First attachment"; "Second attachment" ] + (Atomic.get imports_entered); + wait "first failure exposes Error info after later success" (fun () -> + Option.is_some (find "accessibility-label" "Error info")); + dispatch + (Lui_protocol.Press (Option.get (find "accessibility-label" "Error info"))); + wait "Error info retains the first batch failure" (fun () -> + Option.is_some (find "text" "Fixture first attachment failed"))); if check_ios_capture then ( dispatch @@ -2058,7 +2501,8 @@ let run_favorites_native_visibility (Option.value (Hashtbl.find_opt regions "timeline") ~default:0)); Hashtbl.clear regions; dispatch (Lui_protocol.Press favorites); - wait "Favorites loaded" (fun () -> Option.is_some (find "text" "Fixture 64")); + load_all_favorites "Favorites loaded"; + settle_worker (); if check_chrome then ( Hashtbl.clear regions; @@ -2144,13 +2588,13 @@ let run_favorites_native_visibility if (not check_detail) && not check_generation then ( let show n = - let text = Option.get (find "text" (Printf.sprintf "Fixture %d" n)) in + let text = Option.get (find "text" (favorite_label n)) in dispatch (Lui_protocol.Appear (ancestor_property text "appear-enabled")) in for n = 0 to 63 do show n; wait (Printf.sprintf "root %d query completed" n) (fun () -> - List.mem (List.nth roots n) (Atomic.get queried)); + List.mem (favorite_target_uuid n) (Atomic.get queried)); (* Drain the response as well, so the next root is not query-concurrency limited. *) for _ = 1 to 3 do hooks.pump () |> consume; @@ -2170,7 +2614,7 @@ let run_favorites_native_visibility |> consume; show 64; wait "65th root admitted after real Favorites Int64_pair event" (fun () -> - List.mem (List.nth roots 64) (Atomic.get queried)); + List.mem (favorite_target_uuid 64) (Atomic.get queried)); let list_node = Option.get (find "_extension" "journal-list") in hooks.extension_event list_node @@ -2190,7 +2634,9 @@ let run_favorites_native_visibility "same visible page root does not refetch" 1 (List.length - (List.filter (G.Uuid.equal (List.nth roots 64)) (Atomic.get queried)))))) + (List.filter + (G.Uuid.equal (favorite_target_uuid 64)) + (Atomic.get queried)))))) ;; (* The root reducer has no Worker accepted-ID/terminal-event boundary. This @@ -2309,6 +2755,14 @@ let test_reference_outer_terminals cancel = | Import_asset _ -> Error "unused import" | Graph_request request -> (match request.command with + | P.V2_get_block { block = target; _ } + when not (Array.exists (G.Uuid.equal target) targets) -> + completed + request + (V2_block_outcome + (if G.Uuid.equal target block.uuid + then V2_present_block { value = record block; revision = "root" } + else V2_missing_block { uuid = target; revision = "p" })) | P.V2_get_block { block = target; _ } -> let index = Array.to_list targets @@ -2407,12 +2861,12 @@ let test_reference_outer_terminals cancel = request (V2_changes { generation = "g" - ; from_exclusive = Some "r1" + ; from_exclusive = Some "p" ; through = "r2" ; next = None ; windows = [ { id = "c1" - ; predecessor = "r1" + ; predecessor = "p" ; successor = "r2" ; block_uuids = Array.to_list (Array.sub targets 0 4) ; page_uuids = [] @@ -2455,6 +2909,7 @@ let test_reference_outer_terminals cancel = service in let texts = Hashtbl.create 128 in + let track_teardown = track_json_wire_teardown (Hashtbl.remove texts) in let saw_late = ref false in let consume encoded = if encoded <> "" @@ -2464,6 +2919,7 @@ let test_reference_outer_terminals cancel = |> member "ops" |> to_list |> List.iter (fun op -> + track_teardown op; match op |> member "op" |> to_string with | "set-prop" when member "property" op = `String "text" -> let text = member "value" op |> to_string in @@ -2473,7 +2929,6 @@ let test_reference_outer_terminals cancel = (String.split_on_char '|' text) then saw_late := true; Hashtbl.replace texts (member "id" op |> to_int) text - | "drop-node" -> Hashtbl.remove texts (member "id" op |> to_int) | _ -> ()) in let wait label predicate = @@ -2482,6 +2937,15 @@ let test_reference_outer_terminals cancel = consume (hooks.pump ()); Unix.sleepf 0.001 done; + if not (predicate ()) + then + Printf.eprintf + "reference timeout=%s attempts=[%s] terminals=%d\n%!" + label + (String.concat + ";" + (Array.to_list attempts |> List.map (fun a -> string_of_int (Atomic.get a)))) + !terminal_count; Alcotest.(check bool) label true (predicate ()) in let startup = @@ -2564,17 +3028,19 @@ let with_timeline_media_row run = let entry = { Journal_graph_projection.block; child_summaries = [] } in let batches = ref [] in let texts = Hashtbl.create 32 in + let track_teardown = track_wire_teardown (Hashtbl.remove texts) in let backend : Lui_protocol.backend = { backend_profile = Lui_protocol.profile IOS SwiftUIHost ; apply_batch = (fun batch -> batches := batch :: !batches; List.iter - (function - | Lui_protocol.SetProp (id, TextValue, StringValue text) -> - Hashtbl.replace texts id text - | DropNode id -> Hashtbl.remove texts id - | _ -> ()) + (fun op -> + track_teardown op; + match op with + | Lui_protocol.SetProp (id, TextValue, StringValue text) -> + Hashtbl.replace texts id text + | _ -> ()) batch.ops; true) } @@ -2671,7 +3137,7 @@ let test_timeline_media_identity_across_detail_routes () = let drops = List.filter (function - | Lui_protocol.DropNode _ -> true + | Lui_protocol.DropNode _ | DetachSubtree _ -> true | _ -> false) (ops ()) in @@ -2724,20 +3190,24 @@ let test_timeline_media_disclosure_and_owner_replacement () = let test_reactive_header_current_controls () = let props = Hashtbl.create 128 in + let track_teardown = + track_wire_teardown (fun id -> + Hashtbl.filter_map_inplace + (fun (node, _) value -> if node = id then None else Some value) + props) + in let backend : Lui_protocol.backend = { backend_profile = Lui_protocol.profile IOS SwiftUIHost ; apply_batch = (fun batch -> List.iter - (function - | Lui_protocol.SetProp (id, key, value) -> - Hashtbl.replace props (id, key) value - | RemoveProp (id, key) -> Hashtbl.remove props (id, key) - | DropNode id -> - Hashtbl.filter_map_inplace - (fun (node, _) value -> if node = id then None else Some value) - props - | _ -> ()) + (fun op -> + track_teardown op; + match op with + | Lui_protocol.SetProp (id, key, value) -> + Hashtbl.replace props (id, key) value + | RemoveProp (id, key) -> Hashtbl.remove props (id, key) + | _ -> ()) batch.Lui_protocol.ops; true) } @@ -3029,6 +3499,7 @@ let unlock_fixture () = let test_unlock_application () = let hooks, submissions, release, client = unlock_fixture () in let props = Hashtbl.create 64 in + let track_teardown = track_json_wire_teardown (Hashtbl.remove props) in let consume encoded = if encoded <> "" then @@ -3037,6 +3508,7 @@ let test_unlock_application () = |> member "ops" |> to_list |> List.iter (fun op -> + track_teardown op; let id = op |> member "id" in match op |> member "op" |> to_string with | "create-node" | "create-extension" -> Hashtbl.replace props (to_int id) [] @@ -3048,7 +3520,6 @@ let test_unlock_application () = props id ((key, member "value" op) :: List.remove_assoc key previous) - | "drop-node" | "drop-extension" -> Hashtbl.remove props (to_int id) | _ -> ()) in let find key value = @@ -3393,6 +3864,12 @@ let () = ; Alcotest.test_case "fixture early exit releases Worker" `Quick (fun () -> test_fixture_exception_cleanup Exit) ] ) + ; ( "Capture attachment imports" + , [ Alcotest.test_case + "ordered partial failure and staging release" + `Quick + (fun () -> run_favorites_native_visibility ~check_capture_imports:true ()) + ] ) ; ( "targeted media subscriptions" , [ Alcotest.test_case "single Ready and Acquire N=3" `Quick (fun () -> run_favorites_native_visibility ~media_rows:3 ()) @@ -3517,7 +3994,7 @@ let () = ~media_rows:3 ~media_navigation:`Preview_graph ()) - ; Alcotest.test_case "invalid availability closes preview URL" `Quick (fun () -> + ; Alcotest.test_case "late Failed preserves successful preview URL" `Quick (fun () -> run_favorites_native_visibility ~media_rows:3 ~media_navigation:`Preview_invalidated @@ -3530,7 +4007,7 @@ let () = ~media_rows:3 ~media_navigation:`Preview_repeat ()) - ; Alcotest.test_case "descriptor replacement retires preview" `Quick (fun () -> + ; Alcotest.test_case "same UUID metadata preserves preview" `Quick (fun () -> run_favorites_native_visibility ~media_rows:3 ~media_navigation:`Preview_replaced diff --git a/test/journal_asset_policy_test.ml b/test/journal_asset_policy_test.ml index 9c25154..81d4de3 100644 --- a/test/journal_asset_policy_test.ml +++ b/test/journal_asset_policy_test.ml @@ -90,7 +90,7 @@ let pages () = "continue favorites beyond UI page"; let s, ins = P.step s (Roots_loaded (r, [], None)) in check (ins = [] && P.progress s Favorites = Complete) "complete enumeration"; - let s, ins = P.step s Graph_changed in + let s, ins = P.step s Resync in let r = read Favorites ins in check (not @@ -203,7 +203,7 @@ let residency () = in let status = P.offline s Favorites in check (status.ready = 0 && status.failed = 1) "lost availability revokes completeness"; - let s, ins = P.step s Graph_changed in + let s, ins = P.step s Resync in check ((P.offline s Favorites).enumeration = Enumerating) "replacement cannot claim completeness"; @@ -216,7 +216,7 @@ let residency () = (* Existing Refresh reproduces graph-push replacement in the baseline. GREEN separates this invalidation from unchanged lifecycle configuration. *) -let changed state = P.step state Graph_changed +let changed state = P.step state Resync let read_count ins = List.length @@ -249,7 +249,7 @@ let burst_roots () = for _ = 1 to 32 do let s, ins = changed !state in state := s; - check (read_count ins = 0) "32 graph changes must coalesce behind each root read" + check (read_count ins = 0) "32 resync requests must coalesce behind each root read" done; let s, ins = P.step !state (Roots_loaded (old, [ uuid 9 ], Some cursor)) in let fresh = read Favorites ins in @@ -313,9 +313,7 @@ let committed_and_finished () = (read_count ins = 0 && P.progress s Favorites = Complete) "same configuration must not rescan a finished enumeration"; let s, ins = changed s in - check - (read_count ins = 1) - "actual graph change must restart finished Favorites only once"; + check (read_count ins = 1) "explicit resync must restart finished Favorites only once"; check (not (List.exists @@ -388,6 +386,859 @@ let runtime_same_configuration () = check accepted "adapter must not swallow still-owned response on same configuration" ;; +let completed_favorites () = + let s, ins = refresh P.empty in + let s, ins = P.step s (Roots_loaded (read Favorites ins, [ uuid 1 ], None)) in + let s, ins = P.step s (Assets_loaded (read Favorites ins, [ asset 2 ], None)) in + let consumer, _ = demand ins in + let s, _ = P.step s (Demand_accepted consumer) in + s, consumer +;; + +let unrelated_roots () = + let s, _ = completed_favorites () in + let state = ref s in + for _ = 1 to 55 do + let s, ins = P.step !state (Roots_changed [ uuid 99 ]) in + state := s; + check + (ins = []) + "unrelated roots must preserve completed offline demand without reads" + done +;; + +let related_roots () = + let s, consumer = completed_favorites () in + let s, ins = P.step s (Roots_changed [ uuid 1 ]) in + check (read_count ins = 1) "one related batch only"; + let ticket = read Favorites ins in + check + (ticket.query = Assets { roots = [ uuid 1 ]; cursor = None }) + "membership updates must reread the related batch without index enumeration"; + check (not (List.mem (P.Release consumer) ins)) "old demand remains during replacement"; + let _, ins = P.step s (Assets_loaded (ticket, [], None)) in + check + (List.mem (P.Release consumer) ins) + "empty replacement releases removed attachments" +;; + +let same_index_roots () = + let s, _ = completed_favorites () in + let s, ins = P.step s (Index_changed Favorites) in + check (read_count ins = 1) "only changed index enumerates"; + let _, ins = P.step s (Roots_loaded (read Favorites ins, [ uuid 1 ], None)) in + check (ins = []) "unchanged index roots must not repeat recursive attachment reads" +;; + +let shared_ready_uuid () = + let s, consumer = completed_favorites () in + let s, _ = + P.step s (Availability { consumer; asset = uuid 2; availability = Ready "resident" }) + in + let s, ins = P.step s (Roots_changed [ uuid 1 ]) in + let version = + A.version ~checksum:(String.make 64 'b') ~file_type:"png" |> Result.get_ok + in + let changed = + A.create + ~uuid:(uuid 2) + ~source:(Managed (Some version)) + ~current_checksum:None + ~size:None + ~dimensions:None + |> Result.get_ok + in + let s, ins = P.step s (Assets_loaded (read Favorites ins, [ changed ], None)) in + let fresh, _ = demand ins in + let s, _ = P.step s (Demand_accepted fresh) in + check + ((P.offline s Favorites).ready = 1) + "same resident UUID must not lose readiness on descriptor replacement" +;; + +let exact_batch_and_empty () = + let s, ins = refresh P.empty in + let first_roots = List.init 32 (fun n -> uuid (n + 100)) in + let s, ins = P.step s (Roots_loaded (read Favorites ins, first_roots, Some cursor)) in + let s, ins = P.step s (Assets_loaded (read Favorites ins, [], None)) in + let s, ins = P.step s (Roots_loaded (read Favorites ins, [ uuid 200 ], None)) in + let s, ins = P.step s (Assets_loaded (read Favorites ins, [ asset 300 ], None)) in + let consumer, _ = demand ins in + let s, _ = P.step s (Demand_accepted consumer) in + let s, ins = P.step s (Roots_changed [ uuid 100 ]) in + check (read_count ins = 1) "negative batch dependency causes exactly one read"; + let ticket = read Favorites ins in + check + (ticket.query = Assets { roots = first_roots; cursor = None }) + "preserve exact 32-root bounded owner"; + let s, ins = P.step s (Assets_loaded (ticket, [], None)) in + check + (ins = [] && (P.offline s Favorites).total = 1) + "unrelated committed batch retained after empty replacement" +;; + +let index_diff () = + let s, old = completed_favorites () in + let s, ins = P.step s (Index_changed Favorites) in + let s, ins = P.step s (Roots_loaded (read Favorites ins, [ uuid 3 ], None)) in + let ticket = read Favorites ins in + check + (ticket.query = Assets { roots = [ uuid 3 ]; cursor = None }) + "only added index roots read attachments"; + check + (not (List.mem (P.Release old) ins)) + "removed root demand retained until added roots settle"; + let _, ins = P.step s (Assets_loaded (ticket, [], None)) in + check (List.mem (P.Release old) ins) "removed index root retires its old demand" +;; + +let selected_burst () = + let s, _ = completed_favorites () in + let s, ins = P.step s (Roots_changed [ uuid 1 ]) in + let old = read Favorites ins in + let state = ref s in + for _ = 1 to 32 do + let s, ins = P.step !state (Roots_changed [ uuid 1 ]) in + state := s; + check (ins = []) "selected batch burst holds one pending request" + done; + let s, ins = P.step !state (Assets_loaded (old, [], None)) in + check (read_count ins = 1) "selected batch burst has one bounded followup"; + let _, ins = P.step s (Assets_loaded (read Favorites ins, [], None)) in + check (ins = []) "followup settles without recursive restarts" +;; + +module Runtime = Journal_asset_runtime +module Service = Logseq_db_worker_lui.Logseq_db_worker_lui_service +module Wire = Logseq_db_worker.Protocol + +let block n parent page refs = + G. + { uuid = uuid n + ; title = "title" + ; parent = uuid parent + ; page = uuid page + ; order = "a0" + ; created_at_ms = 0L + ; updated_at_ms = 0L + ; refs = List.map uuid refs + ; tags = [] + ; properties = [] + } +;; + +let present (block : G.block) = + Wire.V2_block_outcome + (V2_present_block + { value = + { block; task_status = None; rendered_page_title = "page"; tag_titles = [] } + ; revision = "r" + }) +;; + +let respond r request_id outcome = + let response = Wire.V2_response { api_version = 2; request_id; outcome } in + check (Runtime.receive r response) "runtime must retain the terminal owner" +;; + +let raw r n (block : G.block) = + let request = + Wire. + { api_version = 2 + ; request_id = uuid n + ; command = V2_get_block { block = block.uuid; revision = None } + } + in + Runtime.observe_request r request; + Runtime.observe_response + r + (V2_response + { api_version = 2; request_id = request.request_id; outcome = present block }) +;; + +let runtime_scope : Service.asset_scope = + { account = + { managed_sync_origin = Uri.of_string "https://sync.example" + ; user_id = "u" + ; account_generation = 1 + ; presentation_generation = 1 + ; lifecycle_generation = 1L + } + ; graph_id = uuid 3 + ; graph_generation = 1 + } +;; + +let runtime_fixture + ?(favorite_block = false) + ?(root_parent = 3) + ?(initial_assets = []) + ?(initial_demand = fun _ -> ()) + ?(changed = fun _ _ _ -> ()) + () + = + let sent = Queue.create () in + let r = + Runtime.create + ~send:(fun request -> + Queue.add request sent; + true) + ~changed + in + Runtime.refresh + r + ~graph_generation:1 + ~today:20260301 + ~settings:(P.settings ~recent_days:0 |> Result.get_ok); + let take () = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> failwith "expected graph read" + in + let roots = take () in + respond + r + roots.request_id + (Wire.V2_favorites_outcome + { favorites_page = Some (uuid 9) + ; generation = "g" + ; projection_revision = "r" + ; next_cursor = None + ; items = + [ { membership_uuid = uuid 10 + ; membership_order = "a0" + ; membership_revision = "r" + ; target = + (if favorite_block + then + V2_favorite_block + { uuid = uuid 2 + ; title = "block" + ; task_status = None + ; revision = "r" + } + else V2_favorite_page { uuid = uuid 1; title = "page"; revision = "r" }) + } + ] + }); + while not (Queue.is_empty sent) do + match Queue.take sent with + | Service.Asset_command { command = Replace_asset_demand { consumer; _ }; _ } -> + initial_demand consumer; + Runtime.notice r runtime_scope (Asset_demand_accepted consumer) + | Service.Graph_request request -> + (match request.command with + | Wire.V2_list_assets _ -> + respond + r + request.request_id + (V2_assets_outcome + { generation = "g" + ; projection_revision = "r" + ; items = initial_assets + ; next_cursor = None + }) + | V2_get_page { page; _ } -> + respond + r + request.request_id + (V2_page_outcome + (V2_present_page + { page = + G. + { uuid = page + ; name = "page" + ; title = "page" + ; kind = Ordinary_page + ; created_at_ms = 0L + ; updated_at_ms = 0L + ; tags = [] + ; properties = [] + ; recycled = false + } + ; revision = "r" + })) + | V2_get_block { block = target; _ } -> + respond + r + request.request_id + (if G.Uuid.equal target (uuid 2) + then present (block 2 root_parent 1 []) + else if G.Uuid.equal target (uuid 3) + then present (block 3 1 1 []) + else V2_block_outcome (V2_missing_block { uuid = target; revision = "r" })) + | _ -> failwith "unexpected initial dependency") + | _ -> () + done; + r, sent +;; + +let window n uuids interests = + Wire. + { id = string_of_int n + ; predecessor = "r" + ; successor = "next" + ; block_uuids = List.map uuid uuids + ; page_uuids = [] + ; structure_interests = interests + } +;; + +let runtime_unconfigured_changes after_shutdown () = + let sent = Queue.create () in + let r = + Runtime.create + ~send:(fun request -> Queue.add request sent; true) + ~changed:(fun _ _ _ -> ()) + in + if after_shutdown then ( + Runtime.refresh + r + ~graph_generation:1 + ~today:20260301 + ~settings:(P.settings ~recent_days:0 |> Result.get_ok); + Runtime.shutdown r; + Queue.clear sent); + (* Graph owns this inline-reference source and its target reads. No offline + configuration has admitted any recursive root or attachment consumer. *) + raw r 600 (block 200 1 1 []); + for n = 1 to 4 do + let request = + Wire. + { api_version = 2 + ; request_id = uuid (600 + n) + ; command = V2_get_block { block = uuid n; revision = None } + } + in + Runtime.observe_request r request; + Runtime.forget_request r ~request_id:request.request_id + done; + Runtime.changes r [ window 1 [ 1; 2; 3; 4 ] [] ]; + Runtime.pump r; + check + (Queue.is_empty sent) + "unconfigured offline owner must not duplicate Graph reference target reads"; + Runtime.changes + r + [ { (window 2 [] [ Wire.V2_children_interest (uuid 99) ]) with + page_uuids = [ uuid 98 ] + } + ]; + Runtime.pump r; + check + (Queue.is_empty sent) + "unconfigured offline owner must not discover structure or page dependencies" +;; + +let runtime_title_only () = + let r, sent = runtime_fixture () in + let original = block 2 1 1 [] in + raw r 600 original; + Runtime.changes r [ window 1 [ 2 ] [] ]; + let request = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> assert false + in + respond r request.request_id (present { original with title = "edited" }); + check (Queue.is_empty sent) "title-only raw delta must not issue an offline asset query" +;; + +let runtime_unknown_unrelated () = + let r, sent = runtime_fixture () in + for n = 1 to 55 do + Runtime.changes r [ window n [ 99 ] [] ]; + let rounds = ref 0 in + while not (Queue.is_empty sent) do + incr rounds; + check (!rounds <= 4) "unknown unrelated holder lookup must terminate"; + match Queue.take sent with + | Service.Graph_request + ({ command = Wire.V2_get_block { block = target; _ }; _ } as request) -> + respond + r + request.request_id + (if G.Uuid.equal target (uuid 99) + then present (block 99 98 98 []) + else V2_block_outcome (V2_missing_block { uuid = target; revision = "r" })) + | _ -> failwith "55 unknown unrelated deltas must not issue an asset/index read" + done + done +;; + +let runtime_refs_delta () = + let r, sent = runtime_fixture () in + raw r 600 (block 2 1 1 []); + Runtime.changes r [ window 1 [ 2 ] [] ]; + let request = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> assert false + in + respond r request.request_id (present (block 2 1 1 [ 50 ])); + let request = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> assert false + in + check + (match request.command with + | V2_list_assets { roots; recursive = true; cursor = None; _ } -> roots = [ uuid 1 ] + | _ -> false) + "raw ref insertion reads only the matching offline batch"; + respond + r + request.request_id + (V2_assets_outcome + { generation = "g"; projection_revision = "r"; items = []; next_cursor = None }); + check (Queue.is_empty sent) "raw ref replacement settles" +;; + +let runtime_sibling_structure () = + let r, sent = runtime_fixture () in + raw r 600 (block 2 1 1 []); + Runtime.changes r [ window 1 [] [ Wire.V2_children_interest (uuid 98) ] ]; + let request = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> assert false + in + respond + r + request.request_id + (V2_block_outcome (V2_missing_block { uuid = uuid 98; revision = "r" })); + check + (Queue.is_empty sent) + "unrelated unknown children scope point read never becomes an offline assets/index \ + read" +;; + +let runtime_unknown_ancestor () = + let r, sent = runtime_fixture ~favorite_block:true () in + raw r 700 (block 2 4 1 []); + Runtime.changes r [ window 1 [ 4 ] [] ]; + let request = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> assert false + in + respond + r + request.request_id + (V2_block_outcome (V2_missing_block { uuid = uuid 4; revision = "r" })); + let fresh = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> failwith "changed ancestor must follow its pre-change query" + in + respond + r + fresh.request_id + (V2_block_outcome (V2_missing_block { uuid = uuid 4; revision = "r" })); + let request = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> failwith "unknown ancestor deletion must replace its dependent root batch" + in + check + (match request.command with + | V2_list_assets { roots; cursor = None; _ } -> roots = [ uuid 2 ] + | _ -> false) + "ancestor liveness has a precise favorite-root dependency" +;; + +let runtime_late_fact () = + let r, sent = runtime_fixture () in + let old = block 2 1 1 [] in + raw r 600 old; + let late_request = + Wire. + { api_version = 2 + ; request_id = uuid 601 + ; command = V2_get_block { block = uuid 2; revision = None } + } + in + Runtime.observe_request r late_request; + Runtime.changes r [ window 1 [ 2 ] [] ]; + Runtime.observe_response + r + (V2_response + { api_version = 2; request_id = late_request.request_id; outcome = present old }); + let request = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> assert false + in + respond r request.request_id (present (block 2 1 1 [ 50 ])); + let request = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> failwith "late pre-change fact must not consume the dirty owner" + in + check + (match request.command with + | V2_list_assets _ -> true + | _ -> false) + "new membership survives a late old raw completion" +;; + +let runtime_known_asset_immutable () = + let r, sent = runtime_fixture () in + let request = + Wire. + { api_version = 2 + ; request_id = uuid 600 + ; command = V2_get_asset_descriptors { assets = [ uuid 50 ] } + } + in + Runtime.observe_request r request; + Runtime.observe_response + r + (V2_response + { api_version = 2 + ; request_id = request.request_id + ; outcome = + V2_assets_outcome + { generation = "g" + ; projection_revision = "r" + ; items = [ asset 50 ] + ; next_cursor = None + } + }); + Runtime.changes r [ window 1 [ 50 ] [] ]; + check + (Queue.is_empty sent) + "same known asset UUID metadata does not renew offline demand" +;; + +let runtime_real_capture_sibling () = + let r, sent = runtime_fixture ~favorite_block:true () in + for n = 1 to 55 do + Runtime.changes + r + [ Wire. + { id = string_of_int n + ; predecessor = "r" + ; successor = "next" + ; block_uuids = [ uuid 7 ] + ; page_uuids = [ uuid 1 ] + ; structure_interests = + [ V2_children_interest (uuid 1); V2_page_tree_interest (uuid 1) ] + } + ]; + let rounds = ref 0 in + while not (Queue.is_empty sent) do + incr rounds; + check (!rounds <= 2) "Capture dependency point reads remain bounded"; + match Queue.take sent with + | Service.Graph_request ({ command = V2_get_block _; _ } as request) -> + respond r request.request_id (present (block 7 1 1 [])) + | Service.Graph_request ({ command = V2_get_page { page; _ }; _ } as request) -> + respond + r + request.request_id + (V2_page_outcome + (V2_present_page + { page = + G. + { uuid = page + ; name = "page" + ; title = "page" + ; kind = Ordinary_page + ; created_at_ms = 0L + ; updated_at_ms = 0L + ; tags = [] + ; properties = [] + ; recycled = false + } + ; revision = "r" + })) + | _ -> + failwith + "55 actual Capture page/tree windows must not reread a sibling favorite \ + block's assets/index" + done + done +;; + +let runtime_registry_capacity () = + let status = ref P.Inactive in + let r, sent = + runtime_fixture ~changed:(fun _ _ favorites -> status := favorites.P.enumeration) () + in + for n = 1000 to 5199 do + Runtime.observe_request + r + Wire. + { api_version = 2 + ; request_id = uuid n + ; command = V2_get_block { block = uuid 99; revision = None } + } + done; + check + (!status = P.Failed) + "4096 accepted nonterminal registrations expose capacity failure"; + check (Queue.is_empty sent) "capacity failure cannot trigger an unbounded rescan"; + Runtime.resync r; + let request = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> assert false + in + respond + r + request.request_id + (V2_favorites_outcome + { favorites_page = Some (uuid 9) + ; generation = "g" + ; projection_revision = "r" + ; items = [] + ; next_cursor = None + }); + check (Queue.is_empty sent) "bounded registry preserves real resync terminal ownership" +;; + +let dependency_capacity_status () = + let s, consumer = completed_favorites () in + let s, _ = + P.step s (Availability { consumer; asset = uuid 2; availability = Ready "resident" }) + in + let s, ins = P.step s Dependencies_unavailable in + check + (ins = [] && P.progress s Favorites = Failed && (P.offline s Favorites).ready = 1) + "dependency capacity failure preserves verified readiness without a fallback read"; + let _, ins = P.step s Shutdown in + check + (List.mem (P.Release consumer) ins) + "capacity failure must preserve and eventually release its existing lease" +;; + +let index_pagination () = + let s, old = completed_favorites () in + let s, ins = P.step s (Index_changed Favorites) in + let s, ins = P.step s (Roots_loaded (read Favorites ins, [ uuid 1 ], Some cursor)) in + let ticket = read Favorites ins in + check + (ticket.query = Favorite_roots (Some cursor)) + "root diff waits for the complete changed index"; + let s, ins = P.step s (Roots_loaded (ticket, [ uuid 3 ], None)) in + let ticket = read Favorites ins in + check + (ticket.query = Assets { roots = [ uuid 3 ]; cursor = None }) + "only added paginated index member reads assets"; + let s, ins = P.step s (Assets_loaded (ticket, [], None)) in + check + ((not (List.mem (P.Release old) ins)) && (P.offline s Favorites).total = 1) + "unchanged roots retain committed offline demand" +;; + +let pending_preserves_ready () = + let s, consumer = completed_favorites () in + let s, _ = + P.step s (Availability { consumer; asset = uuid 2; availability = Ready "resident" }) + in + let s, _ = P.step s (Roots_changed [ uuid 1 ]) in + check + ((P.offline s Favorites).ready = 1) + "bounded membership read does not revoke a resident immutable UUID" +;; + +let runtime_reference_order () = + let r, sent = runtime_fixture () in + raw r 600 (block 2 1 1 [ 50; 51 ]); + Runtime.changes r [ window 1 [ 2 ] [] ]; + let request = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> assert false + in + respond r request.request_id (present (block 2 1 1 [ 51; 50 ])); + check (Queue.is_empty sent) "reference membership order cannot renew offline demand" +;; + +let runtime_hidden_holder_fanout () = + let r, sent = runtime_fixture () in + (* Initial recursive enumeration is empty; no raw fact for holder H exists. + A's class declaration changes on a separate assets page. *) + Runtime.changes r [ window 1 [ 50; 2 ] [] ]; + let queried = ref false in + while not (Queue.is_empty sent) do + match Queue.take sent with + | Service.Graph_request + ({ command = V2_get_block { block = target; _ }; _ } as request) -> + respond + r + request.request_id + (if G.Uuid.equal target (uuid 50) + then present (block 50 99 99 []) + else if G.Uuid.equal target (uuid 2) + then present (block 2 1 1 [ 50 ]) + else V2_block_outcome (V2_missing_block { uuid = target; revision = "r" })) + | Service.Graph_request ({ command = V2_list_assets { roots; _ }; _ } as request) -> + queried := roots = [ uuid 1 ]; + respond + r + request.request_id + (V2_assets_outcome + { generation = "g" + ; projection_revision = "r" + ; items = [ asset 50 ] + ; next_cursor = None + }) + | Service.Asset_command { command = Replace_asset_demand { consumer; _ }; _ } -> + Runtime.notice + r + { account = + { managed_sync_origin = Uri.of_string "https://sync.example" + ; user_id = "u" + ; account_generation = 1 + ; presentation_generation = 1 + ; lifecycle_generation = 1L + } + ; graph_id = uuid 3 + ; graph_generation = 1 + } + (Asset_demand_accepted consumer) + | _ -> () + done; + check + !queried + "new asset declaration needs bounded inverse H fanout; no H raw fact was seeded" +;; + +let runtime_truncated_membership () = + let r, sent = runtime_fixture () in + let holder = + { (block 2 1 1 []) with + properties = + [ G. + { ident = "assets" + ; uuid = uuid 4 + ; title = "Assets" + ; schema = + { property_type = Asset + ; cardinality = Many + ; hidden = false + ; public = true + } + ; values = [ Asset_value (uuid 50) ] + ; values_truncated = true + } + ] + } + in + raw r 600 holder; + Runtime.changes r [ window 1 [ 2 ] [] ]; + let request = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> assert false + in + respond r request.request_id (present holder); + let request = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> failwith "truncated membership cannot prove unchanged hidden references" + in + check + (match request.command with + | V2_list_assets { roots; cursor = None; _ } -> roots = [ uuid 1 ] + | _ -> false) + "truncated owned holder refresh selects only its offline batch" +;; + +let ready_index_retirement () = + let s, consumer = completed_favorites () in + let s, _ = + P.step s (Availability { consumer; asset = uuid 2; availability = Ready "resident" }) + in + let s, ins = P.step s (Roots_changed [ uuid 1 ]) in + let s, _ = P.step s (Assets_loaded (read Favorites ins, [], None)) in + let s, _ = + P.step s (Availability { consumer; asset = uuid 2; availability = Ready "late" }) + in + let s, ins = P.step s (Roots_changed [ uuid 1 ]) in + let s, ins = P.step s (Assets_loaded (read Favorites ins, [ asset 2 ], None)) in + let fresh, _ = demand ins in + let s, _ = P.step s (Demand_accepted fresh) in + check + ((P.offline s Favorites).ready = 0) + "last consumer release retires successful UUID inheritance; late notice cannot \ + revive it" +;; + +let runtime_unknown_old_parent () = + let old = ref None + and released = ref false in + let r, sent = + runtime_fixture + ~favorite_block:true + ~root_parent:1 + ~initial_assets:[ asset 50 ] + ~initial_demand:(fun consumer -> old := Some consumer) + () + in + Runtime.changes + r + [ window + 1 + [ 4 ] + [ V2_children_interest (uuid 3) + ; V2_children_interest (uuid 7) + ; V2_page_tree_interest (uuid 1) + ] + ]; + let queried = ref false in + while not (Queue.is_empty sent) do + match Queue.take sent with + | Service.Graph_request + ({ command = V2_get_block { block = target; _ }; _ } as request) -> + respond + r + request.request_id + (if G.Uuid.equal target (uuid 4) + then present (block 4 7 1 []) + else if G.Uuid.equal target (uuid 3) + then present (block 3 2 1 []) + else V2_block_outcome (V2_missing_block { uuid = target; revision = "r" })) + | Service.Graph_request ({ command = V2_list_assets { roots; _ }; _ } as request) -> + queried := roots = [ uuid 2 ]; + respond + r + request.request_id + (V2_assets_outcome + { generation = "g"; projection_revision = "r"; items = []; next_cursor = None }) + | Service.Asset_command { command = Release_asset_demand consumer; _ } -> + released := Some consumer = !old + | _ -> () + done; + check + (!queried && !released) + "old unknown structural parent resolves the old batch owner after a hidden child \ + moves out" +;; + +let runtime_dependency_failure rejected () = + let status = ref P.Inactive in + let r, sent = + runtime_fixture ~changed:(fun _ _ favorites -> status := favorites.P.enumeration) () + in + Runtime.changes r [ window 1 [ 99 ] [] ]; + let request = + match Queue.take sent with + | Service.Graph_request request -> request + | _ -> assert false + in + if rejected + then Runtime.reject r ~request_id:request.request_id + else + respond r request.request_id (V2_failed { code = "storage"; message = "unavailable" }); + check + (!status = P.Failed && Queue.is_empty sent) + "owned dependency failure must expose Failed without retrying or preserving false \ + completeness" +;; + let () = let failed = ref 0 in List.iter @@ -399,7 +1250,35 @@ let () = | exn -> incr failed; Printf.printf "FAIL %s: %s\n%!" name (Printexc.to_string exn)) - [ "calendar", interval + [ "runtime never configured changes", runtime_unconfigured_changes false + ; "runtime shutdown changes", runtime_unconfigured_changes true + ; "runtime unknown old parent", runtime_unknown_old_parent + ; "runtime dependency reject", runtime_dependency_failure true + ; "runtime dependency failed", runtime_dependency_failure false + ; "runtime hidden negative holder", runtime_hidden_holder_fanout + ; "runtime truncated membership", runtime_truncated_membership + ; "ready index retirement", ready_index_retirement + ; "runtime equivalent reference set", runtime_reference_order + ; "immutable pending residency", pending_preserves_ready + ; "runtime actual Capture sibling", runtime_real_capture_sibling + ; "runtime accepted registry capacity", runtime_registry_capacity + ; "dependency capacity preserves lease", dependency_capacity_status + ; "immutable paginated index diff", index_pagination + ; "runtime unknown ancestor", runtime_unknown_ancestor + ; "runtime late raw fact", runtime_late_fact + ; "runtime immutable asset UUID", runtime_known_asset_immutable + ; "runtime title-only raw facts", runtime_title_only + ; "runtime unknown unrelated 55", runtime_unknown_unrelated + ; "runtime raw ref insertion", runtime_refs_delta + ; "runtime unrelated structure", runtime_sibling_structure + ; "immutable resident UUID", shared_ready_uuid + ; "immutable exact negative batch", exact_batch_and_empty + ; "immutable index root diff", index_diff + ; "immutable selected burst 32", selected_burst + ; "immutable unrelated 55", unrelated_roots + ; "immutable related batch", related_roots + ; "immutable unchanged index", same_index_roots + ; "calendar", interval ; "paginated replacement", pages ; "fencing", fencing ; "visible", visible diff --git a/test/journal_graph_runtime_locality_test.ml b/test/journal_graph_runtime_locality_test.ml index cbd56f5..b4a6515 100644 --- a/test/journal_graph_runtime_locality_test.ml +++ b/test/journal_graph_runtime_locality_test.ml @@ -478,6 +478,17 @@ let test_resync_and_dependent_reads_share_hydration_bound () = ; items = [] ; next_cursor = None } ) + | V2_get_block { block = id; _ } -> + let value = + List.find + (fun (value : Protocol.v2_block_record) -> + Graph.Uuid.equal value.block.uuid id) + records + in + ( "block" + , Protocol.V2_block_outcome (V2_present_block { value; revision = "block-2" }) ) + | V2_get_page _ -> + "page", Protocol.V2_page_outcome (V2_present_page { page; revision = "page-2" }) | _ -> Alcotest.fail "resync request outside retained interests" in Hashtbl.replace @@ -496,7 +507,7 @@ let test_resync_and_dependent_reads_share_hydration_bound () = kind expected (Option.value ~default:0 (Hashtbl.find_opt counts kind))) - [ "graph", 1; "journal", 1; "tree", 2; "children", 48 ] + [ "graph", 1; "journal", 1; "tree", 2; "children", 48; "block", 48; "page", 1 ] ;; let test_queued_resync_announces_feed_refresh () = @@ -524,7 +535,7 @@ let test_queued_resync_announces_feed_refresh () = (List.length output.requests); Alcotest.(check int) "refresh ownership is announced before queued reads" - 2 + 3 (List.length output.responses); Alcotest.(check bool) "queued refresh retains its request generation" @@ -3161,6 +3172,630 @@ let test_copy_result_fits_serialized_clipboard () = [ maximum - 19; maximum - 18 ] ;; +let seed_ordinary_detail runtime = + set_calendar runtime; + let ordinary_page = + { page with kind = Ordinary_page; name = "reference"; title = "Reference" } + in + let request = + Runtime.submit + runtime + (Journal_graph_request.Load_detail + { block_id = Graph.Uuid.to_string block_uuid + ; after = None + ; limit = 3 + ; request_generation = 91L + }) + |> fun output -> only "ordinary detail root" output.requests + in + let request = + Runtime.receive + runtime + (response + request + (Protocol.V2_block_outcome + (V2_present_block { value = record; revision = "root-1" }))) + |> fun output -> only "ordinary detail page" output.requests + in + let request = + Runtime.receive + runtime + (response + request + (Protocol.V2_page_outcome + (V2_present_page { page = ordinary_page; revision = "page-1" }))) + |> fun output -> only "ordinary detail children" output.requests + in + let child = + { record with + block = { block with uuid = unrelated_uuid; title = "Child"; parent = block_uuid } + } + in + ignore + (Runtime.receive + runtime + (response + request + (Protocol.V2_children_outcome + { parent = block_uuid + ; revision_scope = V2_children_revision block_uuid + ; scope_revision = "children-1" + ; items = [ { value = child; revision = "child-1" } ] + ; next_cursor = None + }))); + ordinary_page, child +;; + +let test_ordinary_detail_point_change child_changed = + let runtime = Runtime.create () in + let _, child = seed_ordinary_detail runtime in + let value = if child_changed then child else record in + let output = changed runtime (window ~blocks:[ value.block.uuid ] ()) in + let request = only "ordinary detail changed identity" (hydration_requests output) in + (match request.command with + | V2_get_block { block; _ } -> + Alcotest.(check bool) + "only changed identity" + true + (Graph.Uuid.equal block value.block.uuid) + | _ -> Alcotest.fail "ordinary detail did not point-read its changed identity"); + let output = + Runtime.receive + runtime + (response + request + (Protocol.V2_block_outcome + (V2_present_block + { value = + { value with block = { value.block with title = "Updated detail" } } + ; revision = "detail-2" + }))) + in + let published = + List.find_map + (fun result -> + match result.Runtime.payload with + | Block_updated { block; _ } -> Some block + | _ -> None) + output.responses + in + match published with + | Some block -> + Alcotest.(check string) + "fresh ordinary detail source" + "Updated detail" + (Journal_model.source block) + | None -> Alcotest.fail "ordinary detail changed identity did not publish" +;; + +let test_detail_children_uses_latest_root () = + let runtime = Runtime.create () in + seed_children_interest runtime; + let output = + changed + runtime + (window + ~blocks:[ block_uuid ] + ~scopes:[ Protocol.V2_children_interest block_uuid ] + ()) + in + let point = + List.find + (fun (request : Protocol.request) -> + match request.command with + | V2_get_block _ -> true + | _ -> false) + (hydration_requests output) + in + let children = + List.find + (fun (request : Protocol.request) -> + match request.command with + | V2_get_children _ -> true + | _ -> false) + (hydration_requests output) + in + ignore + (Runtime.receive + runtime + (response + point + (Protocol.V2_block_outcome + (V2_present_block + { value = { record with block = { block with title = "Fresh root" } } + ; revision = "root-2" + })))); + let output = + Runtime.receive + runtime + (response + children + (Protocol.V2_children_outcome + { parent = block_uuid + ; revision_scope = V2_children_revision block_uuid + ; scope_revision = "children-2" + ; items = [] + ; next_cursor = None + })) + in + match + List.find_map + (fun result -> + match result.Runtime.payload with + | Children_reconciled detail -> Some detail + | _ -> None) + output.responses + with + | Some detail -> + Alcotest.(check string) + "children must not restore stale root" + "Fresh root" + (Journal_model.source detail.root) + | None -> Alcotest.fail "changed children were not reconciled" +;; + +let test_detail_resync_reads_latest_root () = + let runtime = Runtime.create () in + ignore (seed_ordinary_detail runtime); + let output = + Runtime.reconcile_push + runtime + ~request_generation:12L + (Protocol.V2_resync_required_push + { api_version = Protocol.api_version + ; generation = "generation-2" + ; reason = "overflow" + }) + in + Alcotest.(check bool) + "resync must rehydrate ordinary detail root content" + true + (List.exists + (fun (request : Protocol.request) -> + match request.command with + | V2_get_block { block; _ } -> Graph.Uuid.equal block block_uuid + | _ -> false) + output.requests) +;; + +let favorites_page_uuid = uuid "a1000000-0000-4000-8000-000000000009" +let membership_uuid = uuid "a1000000-0000-4000-9000-000000000009" +let ancestor_uuid = uuid "a1000000-0000-4000-9000-000000000008" + +let seed_favorite_dependencies + ?(negative = false) + ?(before_target = fun () -> ()) + ?(target_title = fun _ -> block.title) + ?(on_output = fun (_ : Runtime.output) -> ()) + runtime + = + set_calendar runtime; + let request : Journal_graph_request.favorites_request = + { graph_generation = 1; request_generation = 1L; limit = 50; cursor = None } + in + let read = + Runtime.submit runtime (Journal_graph_request.Load_favorites request) + |> fun output -> only "favorites read" output.requests + in + let output = + Runtime.receive + runtime + (response + read + (Protocol.V2_favorites_outcome + { favorites_page = Some favorites_page_uuid + ; generation = "generation-1" + ; projection_revision = "revision-1" + ; items = + (if negative + then [] + else + [ { membership_uuid + ; membership_order = "a" + ; membership_revision = "membership-1" + ; target = + V2_favorite_block + { uuid = block_uuid + ; title = block.title + ; task_status = None + ; revision = "root-1" + } + } + ]) + ; next_cursor = None + })) + in + let visited = ref [] in + let target_reads = ref 0 in + let rec drain requests = + match requests with + | [] -> () + | (request : Protocol.request) :: rest -> + let outcome = + match request.command with + | V2_get_children { parent; _ } when Graph.Uuid.equal parent favorites_page_uuid + -> + Protocol.V2_children_outcome + { parent + ; revision_scope = V2_children_revision parent + ; scope_revision = "favorites-memberships-1" + ; items = + [ { value = + { record with + block = + { block with uuid = membership_uuid; parent; page = parent } + } + ; revision = "membership-1" + } + ] + ; next_cursor = None + } + | V2_get_block { block = id; _ } -> + visited := Graph.Uuid.to_string id :: !visited; + let value = + if Graph.Uuid.equal id block_uuid + then ( + incr target_reads; + if !target_reads = 1 then before_target (); + { record with + block = + { block with + parent = ancestor_uuid + ; title = target_title !target_reads + } + }) + else if Graph.Uuid.equal id ancestor_uuid + then + { record with + block = { block with uuid = ancestor_uuid; parent = page_uuid } + } + else Alcotest.fail "dependency read escaped favorite ancestry" + in + Protocol.V2_block_outcome + (V2_present_block { value; revision = "dependency-1" }) + | V2_get_page { page = id; _ } when Graph.Uuid.equal id page_uuid -> + Protocol.V2_page_outcome (V2_present_page { page; revision = "page-1" }) + | _ -> Alcotest.fail "favorite dependency used an unbounded discovery request" + in + let output = Runtime.receive runtime (response request outcome) in + on_output output; + let pending = rest @ output.requests in + check_hydration_bound pending; + drain pending + in + check_hydration_bound output.requests; + drain output.requests; + !visited +;; + +let invalidates_favorites output = + List.exists + (fun result -> + match result.Runtime.payload with + | Favorites_invalidated -> true + | _ -> false) + output.Runtime.responses +;; + +let test_favorite_dependencies_are_read () = + let runtime = Runtime.create () in + let visited = seed_favorite_dependencies runtime in + Alcotest.(check (list string)) + "favorite target and ancestors resolved using bounded point reads" + (List.sort + String.compare + [ Graph.Uuid.to_string block_uuid; Graph.Uuid.to_string ancestor_uuid ]) + (List.sort_uniq String.compare visited) +;; + +let test_favorite_pending_target_change_is_rechecked () = + let runtime = Runtime.create () in + let invalidated = ref false in + let visited = + seed_favorite_dependencies + runtime + ~before_target:(fun () -> + let output = changed runtime (window ~blocks:[ block_uuid ] ()) in + Alcotest.(check int) + "held point query retains one slot" + 0 + (List.length (hydration_requests output))) + ~target_title:(fun count -> if count = 1 then block.title else "Fresh favorite") + ~on_output:(fun output -> + invalidated := !invalidated || invalidates_favorites output) + in + Alcotest.(check int) + "owned target change rechecks once after old completion" + 2 + (List.length (List.filter (String.equal (Graph.Uuid.to_string block_uuid)) visited)); + Alcotest.(check bool) "fresh target title invalidates old Favorites" true !invalidated +;; + +let test_negative_favorite_membership_intersects () = + let runtime = Runtime.create () in + ignore (seed_favorite_dependencies ~negative:true runtime); + Alcotest.(check bool) + "negative membership is retained as a dependency" + true + (invalidates_favorites (changed runtime (window ~blocks:[ membership_uuid ] ()))) +;; + +let test_favorite_changes_locality () = + let runtime = Runtime.create () in + ignore (seed_favorite_dependencies runtime); + let request = pull runtime in + let output = + Runtime.receive + runtime + (response + request + (Protocol.V2_changes + { generation = "generation-1" + ; from_exclusive = Some "revision-1" + ; through = "revision-2" + ; windows = [ window ~blocks:[ unrelated_uuid ] () ] + ; next = None + })) + in + Alcotest.(check bool) + "unrelated owned changes preserve favorites" + false + (invalidates_favorites output); + List.iter + (fun window -> + Alcotest.(check bool) + "membership intersection invalidates favorites" + true + (invalidates_favorites (changed runtime window))) + [ window ~blocks:[ membership_uuid ] () + ; window ~scopes:[ Protocol.V2_children_interest favorites_page_uuid ] () + ]; + List.iter + (fun (window, outcome) -> + let runtime = Runtime.create () in + ignore (seed_favorite_dependencies runtime); + let out = changed runtime window in + Alcotest.(check bool) + "dependency facts are rechecked before invalidating" + false + (invalidates_favorites out); + let read = only "bounded dependency recheck" (hydration_requests out) in + let out = Runtime.receive runtime (response read outcome) in + Alcotest.(check bool) + "changed target or eligibility invalidates favorites" + true + (invalidates_favorites out)) + [ ( window ~blocks:[ block_uuid ] () + , Protocol.V2_block_outcome + (V2_present_block + { value = + { record with + block = { block with parent = ancestor_uuid; title = "Fresh favorite" } + } + ; revision = "favorite-2" + }) ) + ; ( window ~blocks:[ ancestor_uuid ] () + , Protocol.V2_block_outcome + (V2_missing_block { uuid = ancestor_uuid; revision = "missing-2" }) ) + ; ( window ~pages:[ page_uuid ] () + , Protocol.V2_page_outcome + (V2_present_page { page = { page with recycled = true }; revision = "page-2" }) + ) + ] +;; + +let test_favorite_sibling_structure_is_local () = + let runtime = Runtime.create () in + ignore (seed_favorite_dependencies runtime); + let output = + changed + runtime + (window + ~blocks:[ unrelated_uuid ] + ~pages:[ page_uuid ] + ~scopes: + [ Protocol.V2_children_interest page_uuid; V2_page_tree_interest page_uuid ] + ()) + in + Alcotest.(check bool) + "Capture sibling window preserves Favorites" + false + (invalidates_favorites output); + List.iter + (fun (request : Protocol.request) -> + match request.command with + | V2_get_page { page = id; _ } when Graph.Uuid.equal id page_uuid -> + let output = + Runtime.receive + runtime + (response + request + (Protocol.V2_page_outcome (V2_present_page { page; revision = "page-2" }))) + in + Alcotest.(check bool) + "unchanged live ancestor preserves Favorites" + false + (invalidates_favorites output) + | _ -> Alcotest.fail "sibling change escaped bounded ancestor point read") + (hydration_requests output) +;; + +let test_favorite_ancestor_title_is_local () = + let runtime = Runtime.create () in + ignore (seed_favorite_dependencies runtime); + let output = changed runtime (window ~blocks:[ ancestor_uuid ] ()) in + Alcotest.(check bool) + "ancestor window rechecks first" + false + (invalidates_favorites output); + let read = only "ancestor point read" (hydration_requests output) in + let output = + Runtime.receive + runtime + (response + read + (Protocol.V2_block_outcome + (V2_present_block + { value = + { record with + block = + { block with + uuid = ancestor_uuid + ; parent = page_uuid + ; title = "Renamed ancestor" + } + } + ; revision = "ancestor-2" + }))) + in + Alcotest.(check bool) + "ancestor title does not alter favorite membership" + false + (invalidates_favorites output) +;; + +let test_favorite_dependency_limit () = + let runtime = Runtime.create () in + set_calendar runtime; + let request : Journal_graph_request.favorites_request = + { graph_generation = 1; request_generation = 1L; limit = 50; cursor = None } + in + let read = + Runtime.submit runtime (Journal_graph_request.Load_favorites request) + |> fun out -> only "favorites" out.requests + in + let output = + Runtime.receive + runtime + (response + read + (Protocol.V2_favorites_outcome + { favorites_page = Some favorites_page_uuid + ; generation = "generation-1" + ; projection_revision = "revision-1" + ; items = [] + ; next_cursor = None + })) + in + let rec drain batch requests = + if batch > 20 then Alcotest.fail "Favorite dependencies exceeded 4096 without failure"; + let read = only "bounded membership page" requests in + let items = + List.init 200 (fun offset -> + let uuid = + uuid (Printf.sprintf "a2000000-0000-4000-8000-%012d" ((batch * 200) + offset)) + in + Protocol. + { value = + { record with + block = + { block with + uuid + ; parent = favorites_page_uuid + ; page = favorites_page_uuid + } + } + ; revision = "member-1" + }) + in + let next_cursor = + Some (Graph.Cursor.of_string (Printf.sprintf "members-%d" batch) |> Result.get_ok) + in + let out = + Runtime.receive + runtime + (response + read + (Protocol.V2_children_outcome + { parent = favorites_page_uuid + ; revision_scope = V2_children_revision favorites_page_uuid + ; scope_revision = "members-1" + ; items + ; next_cursor + })) + in + if + List.exists + (fun result -> + match result.Runtime.payload with + | Rejected _ -> true + | _ -> false) + out.responses + then Alcotest.(check int) "failure stops discovery" 0 (List.length out.requests) + else drain (batch + 1) out.requests + in + drain 0 output.requests +;; + +let test_projection_changes_are_owned () = + let runtime = Runtime.create () in + let output = + Runtime.reconcile_push + runtime + ~request_generation:11L + (Protocol.V2_changes_available + { api_version = Protocol.api_version + ; generation = "generation-1" + ; through = "revision-2" + }) + in + Alcotest.(check bool) + "push emits no premature projection invalidation" + false + (invalidates_favorites output); + let request = only "owned changes pull" output.requests in + let change = window ~blocks:[ unrelated_uuid ] () in + let completion = + response + request + (Protocol.V2_changes + { generation = "generation-1" + ; from_exclusive = None + ; through = "revision-2" + ; windows = [ change ] + ; next = None + }) + in + let output = Runtime.receive runtime completion in + Alcotest.(check int) + "owned changes publish once" + 1 + (List.fold_left + (fun count result -> + match result.Runtime.payload with + | Projection_changes [ actual ] when actual = change -> count + 1 + | _ -> count) + 0 + output.responses); + Alcotest.(check int) + "duplicate completion publishes nothing" + 0 + (List.length (Runtime.receive runtime completion).responses); + let output = + Runtime.reconcile_push + runtime + ~request_generation:12L + (Protocol.V2_resync_required_push + { api_version = Protocol.api_version + ; generation = "generation-2" + ; reason = "overflow" + }) + in + Alcotest.(check bool) + "real resync emits projection resync" + true + (List.exists + (fun result -> + match result.Runtime.payload with + | Projection_resync -> true + | _ -> false) + output.responses); + Alcotest.(check bool) + "real resync invalidates favorites" + true + (invalidates_favorites output) +;; + let () = Alcotest.run "journal graph runtime locality" @@ -3320,6 +3955,52 @@ let () = ; "update conflict", Update_conflict ; "delete conflict", Delete_conflict ] ) + ; ( "projection dependencies" + , [ Alcotest.test_case "ordinary detail root changes" `Quick (fun () -> + test_ordinary_detail_point_change false) + ; Alcotest.test_case "ordinary detail child changes" `Quick (fun () -> + test_ordinary_detail_point_change true) + ; Alcotest.test_case + "children retain latest root" + `Quick + test_detail_children_uses_latest_root + ; Alcotest.test_case + "detail resync refreshes root" + `Quick + test_detail_resync_reads_latest_root + ; Alcotest.test_case + "favorites resolve ancestry" + `Quick + test_favorite_dependencies_are_read + ; Alcotest.test_case + "Favorite pending target changes recheck" + `Quick + test_favorite_pending_target_change_is_rechecked + ; Alcotest.test_case + "negative favorites retain membership dependencies" + `Quick + test_negative_favorite_membership_intersects + ; Alcotest.test_case + "favorites owned dependency intersection" + `Quick + test_favorite_changes_locality + ; Alcotest.test_case + "Favorite Capture sibling structure is local" + `Quick + test_favorite_sibling_structure_is_local + ; Alcotest.test_case + "Favorite ancestor title is local" + `Quick + test_favorite_ancestor_title_is_local + ; Alcotest.test_case + "Favorite dependency limit fails explicitly" + `Quick + test_favorite_dependency_limit + ; Alcotest.test_case + "projection changes ownership and real resync" + `Quick + test_projection_changes_are_owned + ] ) ; ( "detail" , [ Alcotest.test_case "resolve unretained page" diff --git a/test/journal_media_runtime_test.ml b/test/journal_media_runtime_test.ml index 3afe4d1..2166a0e 100644 --- a/test/journal_media_runtime_test.ml +++ b/test/journal_media_runtime_test.ml @@ -372,18 +372,11 @@ let fixture () = runtime, sent, views, admitted ;; -let metadata_requests sent = - Queue.to_seq sent - |> List.of_seq - |> List.filter (function - | _, S.Graph_request _ -> true - | _ -> false) -;; - let take_metadata sent = let rec take () = match Queue.take sent with - | Some ticket, S.Graph_request query -> ticket, query.request_id, query.command + | Some ticket, S.Graph_request ({ command = P.V2_list_assets _; _ } as query) -> + ticket, query.request_id, query.command | _ -> take () in take () @@ -418,302 +411,124 @@ let loaded runtime sent root items = reply runtime (take_metadata sent) items ;; -let first_page = function - | P.V2_list_assets { cursor = None; recursive = false; limit = 16; _ } -> true - | _ -> false +let raw_block ?(refs = []) ?(properties = []) n parent : G.block = + { uuid = uuid n + ; parent = uuid parent + ; page = uuid 900 + ; title = "Block" + ; order = "a" + ; created_at_ms = 0L + ; updated_at_ms = 0L + ; refs + ; tags = [] + ; properties + } ;; -let scale_hidden () = - let runtime, sent, _, _ = fixture () in - for n = 1 to 64 do - let root = G.Uuid.to_string (uuid n) in - loaded runtime sent root []; - if n > 2 then R.root_visible runtime ~root false - done; - Queue.clear sent; - R.refresh runtime; - Printf.printf - "COUNT F2 media retained=64 active=2 invalidation_reads=%d\n%!" - (List.length (metadata_requests sent)); - check - (List.length (metadata_requests sent) = 2) - "64 retained empty roots with two active must produce two reads, not 64"; - for _ = 1 to 32 do - R.refresh runtime - done; - Printf.printf - "COUNT F2 media 32 additional invalidations pending_reads=%d\n%!" - (List.length (metadata_requests sent)); - check - (List.length (metadata_requests sent) = 2) - "32 refreshes cannot replace two in-flight metadata owners" +let block_response block = + P.V2_response + { api_version = 2 + ; request_id = uuid 999 + ; outcome = + V2_block_outcome + (V2_present_block + { value = + { block + ; task_status = None + ; rendered_page_title = "Page" + ; tag_titles = [] + } + ; revision = "r" + }) + } ;; -let pending_followup terminal () = - let runtime, sent, views, _ = fixture () in - let root = G.Uuid.to_string (uuid 1) in - loaded runtime sent root [ media_asset 2 ]; - Queue.clear sent; - R.refresh runtime; - let old = take_metadata sent in - for _ = 1 to 32 do - R.refresh runtime - done; - check (Queue.is_empty sent) "dirty refresh must coalesce behind the current read"; - if terminal - then reply runtime old [ media_asset 3 ] - else - R.reject - runtime - (let t, _, _ = old in - t); - check - ((List.hd (Hashtbl.find views root).items).asset.uuid = uuid 2) - "superseded success/failure must preserve the last accepted view"; - check - (List.length (metadata_requests sent) = 1) - "one terminal response must release the owner and issue one fresh follow-up"; - let fresh = take_metadata sent in - let _, _, command = fresh in - check (first_page command) "follow-up must clear projection-bound cursor"; - reply runtime fresh [ media_asset 4 ]; - R.reject +let observe runtime (block : G.block) = + R.observe_request runtime - (let t, _, _ = old in - t); - check - ((List.hd (Hashtbl.find views root).items).asset.uuid = uuid 4) - "duplicate old terminal cannot reject a newer accepted view"; - check (Queue.is_empty sent) "duplicate terminal cannot cause a retry loop" + { api_version = 2 + ; request_id = uuid 999 + ; command = P.V2_get_block { block = block.G.uuid; revision = None } + }; + R.observe_response runtime (block_response block) ;; -let hidden_dirty activation () = - let runtime, sent, views, _ = fixture () in - let root = G.Uuid.to_string (uuid 1) in - let next = G.Cursor.of_string "projection-bound" |> Result.get_ok in - R.root_visible runtime ~root true; - reply runtime (take_metadata sent) ~next_cursor:next [ media_asset 2 ]; - let token = (List.hd (Hashtbl.find views root).items).token in - R.root_visible runtime ~root false; - Queue.clear sent; - R.refresh runtime; - check (Queue.is_empty sent) "hidden dirty root must not read metadata"; - (match activation with - | `Root -> R.root_visible runtime ~root true - | `Asset -> R.asset_visible runtime ~root ~asset:token true - | `Next -> R.next runtime ~root); - let _, _, command = take_metadata sent in - check (first_page command) "activation/next must restart dirty metadata at first page" +let raw_page recycled : G.page = + { uuid = uuid 900 + ; name = "page" + ; title = "Page" + ; kind = Ordinary_page + ; created_at_ms = 0L + ; updated_at_ms = 0L + ; tags = [] + ; properties = [] + ; recycled + } ;; -let hidden_pending () = - let runtime, sent, _, _ = fixture () in - let root = G.Uuid.to_string (uuid 1) in - R.root_visible runtime ~root true; - let old = take_metadata sent in - R.root_visible runtime ~root false; - R.refresh runtime; - check (Queue.is_empty sent) "hidden pending group cannot receive another read"; - reply runtime old []; - check (Queue.is_empty sent) "hidden old terminal cannot restart enumeration"; - R.root_visible runtime ~root true; - check - (List.length (metadata_requests sent) = 1) - "reactivation must release the hidden pending owner and refresh exactly once" +let reply_page runtime sent recycled = + let ticket, request = Queue.take sent in + match ticket, request with + | Some ticket, S.Graph_request { request_id; command = P.V2_get_page { page; _ }; _ } + when page = uuid 900 -> + R.receive + runtime + ticket + (S.Graph_response + (P.V2_response + { api_version = 2 + ; request_id + ; outcome = + V2_page_outcome + (V2_present_page { page = raw_page recycled; revision = "p" }) + })) + | _ -> failwith "page liveness must use one bounded owned get_page" ;; -let blocked_pending () = - let runtime, sent, _, admitted = fixture () in - admitted := false; - for n = 1 to 32 do - R.root_visible runtime ~root:(G.Uuid.to_string (uuid n)) true - done; - for _ = 1 to 32 do - R.refresh runtime - done; - admitted := true; - R.pump runtime; - check - (List.length (metadata_requests sent) = 32) - "32 blocked root owners must keep one queued read each"; - let old = take_metadata sent in - reply runtime old []; - check - (List.length (metadata_requests sent) = 32) - "dirty queued completion yields one follow-up after admission" +let reply_block runtime sent (block : G.block) = + let ticket, request = Queue.take sent in + match ticket, request with + | ( Some ticket + , S.Graph_request { request_id; command = P.V2_get_block { block = uuid; _ }; _ } ) + when uuid = block.G.uuid -> + let outcome = + match block_response block with + | P.V2_response { outcome; _ } -> outcome + in + R.receive + runtime + ticket + (S.Graph_response (P.V2_response { api_version = 2; request_id; outcome })) + | _ -> failwith "holder/liveness must use its owned bounded get_block" ;; -let preview_dirty () = - let runtime, sent, views, _ = fixture () in - let root = G.Uuid.to_string (uuid 1) in - let asset = media_asset 2 in - let scope : S.asset_scope = - { account = - { managed_sync_origin = Uri.of_string "https://sync.example" - ; user_id = "u" - ; account_generation = 1 - ; presentation_generation = 1 - ; lifecycle_generation = 1L - } - ; graph_id = uuid 3 - ; graph_generation = 1 - } - in - let receipt : Logseq_db_worker.import_receipt = - { operation = uuid 8 - ; graph_generation = 1 - ; scope - ; target = uuid 1 - ; asset - ; file_type = "png" - ; preview = Some ("local-lease", "/staged/import.bin") - } - in - R.imported runtime ~current:true receipt; - R.refresh runtime; - check (Queue.is_empty sent) "hidden local preview must remain lazy"; - let token = (List.hd (Hashtbl.find views root).items).token in - R.preview_visible runtime ~owner:"detail" ~slot:"attachment" ~root ~asset:token true; - check - (List.length (metadata_requests sent) = 1) - "preview-only activation must refresh dirty root"; - check - ((List.hd (Hashtbl.find views root).items).presentation - = Journal_media.File "/staged/import.bin") - "invalidation cannot release the local preview lease"; - R.refresh runtime; - check (List.length (metadata_requests sent) = 1) "preview owner remains single-flight" +let window ?(structures = []) blocks : P.v2_change_window = + { id = "change" + ; predecessor = "before" + ; successor = "after" + ; block_uuids = List.map uuid blocks + ; page_uuids = [] + ; structure_interests = structures + } ;; -let bounded_failure () = - let runtime, sent, views, _ = fixture () in - let root = G.Uuid.to_string (uuid 1) in - R.root_visible runtime ~root true; - R.reject - runtime - (let t, _, _ = take_metadata sent in - t); - check (Queue.is_empty sent) "ordinary first-page failure cannot retry indefinitely"; - check - (Option.is_some (Hashtbl.find views root).error) - "ordinary failure remains observable"; - R.reset runtime ~graph_generation:(Some 2); - R.root_visible runtime ~root true; - let old = take_metadata sent in - R.reset runtime ~graph_generation:(Some 3); - R.root_visible runtime ~root true; - let fresh = take_metadata sent in - reply runtime old [ media_asset 3 ]; - reply runtime fresh [ media_asset 4 ]; - check - ((List.hd (Hashtbl.find views root).items).asset.uuid = uuid 4) - "graph reset fences old metadata owners" -;; +let change runtime blocks = R.changes runtime [ window blocks ] -let empty_to_asset () = - let runtime, sent, views, _ = fixture () in - let root = G.Uuid.to_string (uuid 1) in - loaded runtime sent root []; - R.refresh runtime; - reply runtime (take_metadata sent) [ media_asset 7 ]; - check - ((List.hd (Hashtbl.find views root).items).asset.uuid = uuid 7) - "empty result is an unknown dependency, so active invalidation must reveal newly \ - eligible asset" +let seed runtime sent n items = + observe runtime (raw_block n 900); + loaded runtime sent (G.Uuid.to_string (uuid n)) items ;; -let descriptor_lease changed_descriptor () = - let runtime, sent, views, _ = fixture () in - let root = G.Uuid.to_string (uuid 1) in - let descriptor checksum = - A.create - ~uuid:(uuid 2) - ~source: - (Managed - (Some - (A.version ~checksum:(String.make 64 checksum) ~file_type:"png" - |> Result.get_ok))) - ~current_checksum:None - ~size:None - ~dimensions:None - |> Result.get_ok - in - let original = descriptor 'a' in - loaded runtime sent root [ original ]; - let token = (List.hd (Hashtbl.find views root).items).token in - R.asset_visible runtime ~root ~asset:token true; - Queue.clear sent; - let scope : S.asset_scope = - { account = - { managed_sync_origin = Uri.of_string "https://sync.example" - ; user_id = "u" - ; account_generation = 1 - ; presentation_generation = 1 - ; lifecycle_generation = 1L - } - ; graph_id = uuid 3 - ; graph_generation = 1 - } - in - R.notice - runtime - scope - (Asset_availability - { consumer = token; asset = original.uuid; availability = Ready "cached" }); - let ticket, _ = Queue.take sent in - R.receive - runtime - (Option.get ticket) - (Asset_file (Some ("kept-file", "/cache/old.png"))); - R.preview_visible runtime ~owner:"detail" ~slot:"attachment" ~root ~asset:token true; - Queue.clear sent; - R.refresh runtime; - let old = take_metadata sent in - let expected = if changed_descriptor then descriptor 'b' else original in - if changed_descriptor - then ( - R.refresh runtime; - reply runtime old [ expected ]; - check - ((List.hd (Hashtbl.find views root).items).presentation - = Journal_media.File "/cache/old.png") - "superseded same-UUID metadata must not replace the preview's retained file"; - reply runtime (take_metadata sent) [ expected ]) - else reply runtime old [ expected ]; - let item = List.hd (Hashtbl.find views root).items in - check - (item.asset = expected) - "same asset UUID must publish the newly accepted descriptor"; - let releases = - Queue.to_seq sent - |> Seq.filter (function - | _, S.Release_asset_file { handle = "kept-file"; _ } -> true - | _ -> false) - |> Seq.length - in - if changed_descriptor - then - check - (releases = 1 && item.token <> token) - "new source version retires the old controller lease exactly once" - else - check - (releases = 0 - && item.token = token - && item.presentation = Journal_media.File "/cache/old.png") - "unchanged descriptor refresh must preserve preview controller and lease" +let membership_change runtime n target = + observe runtime (raw_block ~refs:[ uuid target ] n 900); + change runtime [ n ] ;; -let stale_continuation hidden () = - let runtime, sent, views, _ = fixture () in - let root = G.Uuid.to_string (uuid 1) in - let cursor = G.Cursor.of_string "old-projection" |> Result.get_ok in - R.root_visible runtime ~root true; - reply runtime (take_metadata sent) ~next_cursor:cursor []; - R.next runtime ~root; - let old = take_metadata sent in - if hidden then R.root_visible runtime ~root false; - let stale (ticket, request_id, _) = +let reply_missing runtime sent n = + let ticket, request = Queue.take sent in + match ticket, request with + | Some ticket, S.Graph_request { request_id; command = P.V2_get_block { block; _ }; _ } + when block = uuid n -> R.receive runtime ticket @@ -722,31 +537,28 @@ let stale_continuation hidden () = { api_version = 2 ; request_id ; outcome = - V2_failed { code = "staleReadCursor"; message = "projection changed" } + V2_block_outcome (V2_missing_block { uuid = block; revision = "deleted" }) })) - in - stale old; - if hidden - then ( - check (Queue.is_empty sent) "stale hidden continuation must stay lazy"; - R.root_visible runtime ~root true); - check - (List.length (metadata_requests sent) = 1) - "clean stale continuation must retry exactly once at first page"; - let fresh = take_metadata sent in - let _, _, command = fresh in - check (first_page command) "stale recovery must discard the global projection cursor"; - stale fresh; - check - (Queue.is_empty sent && Option.is_some (Hashtbl.find views root).error) - "fresh first-page stale failure stops and reports an error"; - R.root_visible runtime ~root true; - check - (Queue.is_empty sent) - "same appearance cannot loop after a first-page stale failure" + | _ -> failwith "deletion must complete its owned bounded block lookup" ;; -let () = +let list_reads sent = + Queue.to_seq sent + |> Seq.filter (function + | _, S.Graph_request { command = P.V2_list_assets _; _ } -> true + | _ -> false) + |> Seq.length +;; + +let file_commands sent = + Queue.to_seq sent + |> Seq.filter (function + | _, (S.Asset_command _ | Acquire_asset_file _ | Release_asset_file _) -> true + | _ -> false) + |> Seq.length +;; + +let run_cases cases = let failed = ref 0 in List.iter (fun (name, test) -> @@ -757,21 +569,551 @@ let () = | exn -> incr failed; Printf.printf "FAIL %s: %s\n%!" name (Printexc.to_string exn)) - [ "F2 64 roots active-only/burst", scale_hidden - ; "F2 superseded metadata success", pending_followup true - ; "F2 superseded metadata reject", pending_followup false - ; "F2 hidden root activation", hidden_dirty `Root - ; "F2 child-before-root activation", hidden_dirty `Asset - ; "F2 dirty next clears cursor", hidden_dirty `Next - ; "F2 hidden pending completion", hidden_pending - ; "F2 32 blocked pending owners", blocked_pending - ; "F2 preview-only/local lease", preview_dirty - ; "F2 failure/reset fences", bounded_failure - ; "F2 empty-to-asset unknown reference", empty_to_asset - ; "F2 same-UUID changed descriptor lease", descriptor_lease true - ; "F2 equal descriptor preview lease", descriptor_lease false - ; "F2 active clean stale cursor bounded", stale_continuation false - ; "F2 hidden clean stale cursor lazy", stale_continuation true - ]; - check (!failed = 0) (Printf.sprintf "%d F2 metadata cases failed" !failed) + cases; + check (!failed = 0) (Printf.sprintf "%d media dependency cases failed" !failed) +;; + +let () = + run_cases + [ ( "stale initial holder facts refill without poisoning the root" + , fun () -> + let r, q, v, _ = fixture () in + let root = G.Uuid.to_string (uuid 1) in + loaded r q root []; + change r [ 1 ]; + change r [ 1 ]; + reply_block r q (raw_block 1 900); + check + (Option.is_none (Hashtbl.find v root).error) + "stale successful first facts are superseded, not a failed load"; + reply_block r q (raw_block ~refs:[ uuid 70 ] 1 900); + check (list_reads q = 1) "fresh initial holder facts reveal new membership" ) + ; ( "stale pending holder facts refill once without accepting old edges" + , fun () -> + let r, q, _, _ = fixture () in + seed r q 1 []; + change r [ 1 ]; + change r [ 1 ]; + reply_block r q (raw_block 1 900); + check + (list_reads q = 0) + "stale holder completion cannot enumerate old membership"; + check + (not (Queue.is_empty q)) + "retired stale Facts must refill the latest holder query"; + reply_block r q (raw_block ~refs:[ uuid 70 ] 1 900); + check (list_reads q = 1) "latest holder facts reveal the new asset reference" ) + ; ( "hidden rejected point reads retire when another enqueue prunes them" + , fun () -> + let r, q, _, accepted = fixture () in + seed r q 1 []; + change r [ 1 ]; + reply_block r q (raw_block 1 900); + accepted := false; + change r [ 1 ]; + R.root_visible r ~root:(G.Uuid.to_string (uuid 1)) false; + accepted := true; + seed r q 2 []; + R.root_visible r ~root:(G.Uuid.to_string (uuid 1)) true; + change r [ 1 ]; + check + (not (Queue.is_empty q)) + "hidden unaccepted Facts cannot retain its request slot forever"; + reply_block r q (raw_block ~refs:[ uuid 70 ] 1 900); + check (list_reads q = 1) "reactivated holder can refill pruned Facts" ) + ; ( "holder title-only change cannot enumerate assets" + , fun () -> + let r, q, _, _ = fixture () in + seed r q 1 []; + change r [ 1 ]; + check + (list_reads q = 0) + "holder notification first checks raw membership signature"; + reply_block r q { (raw_block 1 900) with title = "New title" }; + check + (list_reads q = 0) + "title-only holder point response performs no asset enumeration" ) + ; ( "truncated property observation cannot prove unchanged membership" + , fun () -> + let r, q, _, _ = fixture () in + let property : G.property_summary = + { ident = "fixture/asset" + ; uuid = uuid 71 + ; title = "Asset" + ; schema = + { property_type = Asset + ; cardinality = Many + ; hidden = false + ; public = true + } + ; values = [] + ; values_truncated = true + } + in + let block = raw_block ~properties:[ property ] 1 900 in + observe r block; + loaded r q (G.Uuid.to_string (uuid 1)) []; + change r [ 1 ]; + if list_reads q = 0 then reply_block r q block; + check (list_reads q = 1) "truncated values require a bounded membership read" ) + ; ( "page liveness is distinct from sibling growth" + , fun () -> + let r, q, _, _ = fixture () in + seed r q 1 []; + seed r q 2 []; + let change = { (window [ 80 ]) with page_uuids = [ uuid 900 ] } in + R.changes r [ change ]; + reply_page r q false; + check (list_reads q = 0) "unchanged owning page cannot rescan sibling roots"; + R.changes r [ change ]; + reply_page r q true; + check + (list_reads q = 2) + "page recycling refreshes only its retained member roots" ) + ; ( "successful asset root itself performs zero metadata reads" + , fun () -> + let r, q, v, _ = fixture () in + let asset = + A.create + ~uuid:(uuid 1) + ~source: + (Managed + (Some + (A.version ~checksum:(String.make 64 'a') ~file_type:"png" + |> Result.get_ok))) + ~current_checksum:None + ~size:None + ~dimensions:None + |> Result.get_ok + in + let root = G.Uuid.to_string (uuid 1) in + seed r q 1 [ asset ]; + let token = (List.hd (Hashtbl.find v root).items).token in + R.asset_visible r ~root ~asset:token true; + Queue.clear q; + let scope : S.asset_scope = + { account = + { managed_sync_origin = Uri.of_string "https://sync.example" + ; user_id = "u" + ; account_generation = 1 + ; presentation_generation = 1 + ; lifecycle_generation = 1L + } + ; graph_id = uuid 900 + ; graph_generation = 1 + } + in + R.notice + r + scope + (Asset_availability + { consumer = token; asset = asset.uuid; availability = Ready "cached" }); + let ticket, _ = Queue.take q in + R.receive r (Option.get ticket) (Asset_file (Some ("held", "/cache/held"))); + change r [ 1 ]; + check + (Queue.is_empty q) + "successful asset-as-root UUID metadata window performs zero IO"; + observe r (raw_block ~refs:[ uuid 70 ] 1 900); + change r [ 1 ]; + check + (list_reads q = 1) + "positive holder membership change still refreshes an asset root" ) + ; ( "real Capture window leaves 55 sibling roots alone" + , fun () -> + let r, q, _, _ = fixture () in + for n = 1 to 55 do + seed r q n [] + done; + observe r (raw_block 80 900); + let change = + { (window + ~structures: + [ P.V2_children_interest (uuid 900); V2_page_tree_interest (uuid 900) ] + [ 80 ]) + with + page_uuids = [ uuid 900 ] + } + in + R.changes r [ change ]; + check + (list_reads q = 0) + "actual Capture page/structure footprint must not reread siblings"; + reply_page r q false; + check (list_reads q = 0) "live page confirmation must not enumerate 55 siblings" + ) + ; ( "fact cache capacity fails visibly without retrying" + , fun () -> + let r, q, v, _ = fixture () in + for n = 10000 to 14095 do + observe r (raw_block n 900) + done; + let root = G.Uuid.to_string (uuid 1) in + loaded r q root []; + observe r (raw_block 1 900); + check + (Option.is_some (Hashtbl.find v root).error) + "full raw fact cache must report capacity failure to its affected root"; + for _ = 1 to 32 do + R.pump r + done; + R.root_visible r ~root true; + R.next r ~root; + check + (Queue.is_empty q) + "capacity failure cannot automatically look up the same missing fact or \ + re-read on appearance" ) + ; ( "unrelated long-lived changes preserve current query registration" + , fun () -> + let r, q, _, _ = fixture () in + seed r q 1 []; + R.observe_request + r + { P.api_version = 2 + ; request_id = uuid 998 + ; command = P.V2_get_block { block = uuid 1; revision = None } + }; + for n = 10000 to 14100 do + change r [ n ] + done; + let answer = + match block_response (raw_block ~refs:[ uuid 70 ] 1 900) with + | P.V2_response response -> + P.V2_response { response with request_id = uuid 998 } + in + R.observe_response r answer; + change r [ 70 ]; + check + (list_reads q = 1) + "unrelated capacity pressure cannot clear current-scope query registrations" + ) + ; ( "moved root refreshes raw parent before later ancestor change" + , fun () -> + let r, q, _, _ = fixture () in + seed r q 1 []; + observe r (raw_block 60 900); + change r [ 1 ]; + reply_block r q (raw_block 1 60); + reply r (take_metadata q) []; + change r [ 60 ]; + reply_missing r q 60; + check (list_reads q = 1) "new parent deletion still selects moved root" ) + ; ( "dependency failure stays visible until explicit Retry" + , fun () -> + let r, q, v, _ = fixture () in + observe r (raw_block 1 60); + loaded r q (G.Uuid.to_string (uuid 1)) []; + change r [ 80 ]; + let ticket, request = Queue.take q in + (match ticket, request with + | Some ticket, S.Graph_request { command = P.V2_get_block { block; _ }; _ } + when block = uuid 60 -> R.reject r ticket + | _ -> failwith "missing ancestor must have one owned fact read"); + let root = G.Uuid.to_string (uuid 1) in + check + (Option.is_some (Hashtbl.find v root).error) + "failed dependency lookup is visible"; + R.pump r; + change r [ 81 ]; + check + (Queue.is_empty q) + "unrelated changes cannot automatically retry failed dependency lookup"; + R.retry r ~root ~asset:""; + check (not (Queue.is_empty q)) "explicit Retry can retry the dependency lookup" + ) + ; ( "late owned raw response cannot revert reference facts" + , fun () -> + let r, q, _, _ = fixture () in + seed r q 1 []; + let request id = + { P.api_version = 2 + ; request_id = uuid id + ; command = P.V2_get_block { block = uuid 1; revision = None } + } + in + let answer id refs = + match block_response (raw_block ~refs 1 900) with + | P.V2_response response -> + P.V2_response { response with request_id = uuid id } + in + R.observe_request r (request 997); + R.observe_request r (request 998); + R.observe_response r (answer 998 [ uuid 71 ]); + R.observe_response r (answer 997 [ uuid 70 ]); + change r [ 70 ]; + check (list_reads q = 0) "older owned response cannot restore removed ref"; + change r [ 71 ]; + check (list_reads q = 1) "newest observed ref still selects membership" ) + ; ( "unregistered raw response cannot inject dependencies" + , fun () -> + let r, q, _, _ = fixture () in + seed r q 1 []; + R.observe_response r (block_response (raw_block ~refs:[ uuid 70 ] 1 900)); + change r [ 70 ]; + check (list_reads q = 0) "only registered current-scope facts are accepted" ) + ; ( "unrelated change does not stale in-flight root facts" + , fun () -> + let r, q, _, _ = fixture () in + seed r q 1 []; + R.observe_request + r + { P.api_version = 2 + ; request_id = uuid 998 + ; command = P.V2_get_block { block = uuid 1; revision = None } + }; + change r [ 80 ]; + let answer = + match block_response (raw_block ~refs:[ uuid 70 ] 1 900) with + | P.V2_response response -> + P.V2_response { response with request_id = uuid 998 } + in + R.observe_response r answer; + change r [ 70 ]; + check + (list_reads q = 1) + "unrelated Capture cannot invalidate root fact issuance" ) + ; ( "unrelated Capture leaves 55 roots alone" + , fun () -> + let r, q, _, _ = fixture () in + for n = 1 to 55 do + seed r q n [] + done; + observe r (raw_block 80 900); + change r [ 80 ]; + check (list_reads q = 0) "unrelated Capture cannot enumerate 55 active roots" ) + ; ( "changed holder reveals new membership" + , fun () -> + let r, q, v, _ = fixture () in + seed r q 1 []; + observe r (raw_block ~refs:[ uuid 2 ] 1 900); + change r [ 1 ]; + check (list_reads q = 1) "changed holder must read membership once"; + reply r (take_metadata q) [ media_asset 2 ]; + check + (List.length (Hashtbl.find v (G.Uuid.to_string (uuid 1))).items = 1) + "new member is displayed" ) + ; ( "known foreign ref changes select holder only" + , fun () -> + let r, q, _, _ = fixture () in + observe r (raw_block ~refs:[ uuid 70 ] 1 900); + loaded r q (G.Uuid.to_string (uuid 1)) []; + seed r q 2 []; + change r [ 70 ]; + check (list_reads q = 1) "foreign ref changes only its dependent holder" ) + ; ( "typed property reference is a dependency" + , fun () -> + let r, q, _, _ = fixture () in + let property : G.property_summary = + { ident = "fixture/asset" + ; uuid = uuid 71 + ; title = "Asset" + ; schema = + { property_type = Asset + ; cardinality = One + ; hidden = false + ; public = true + } + ; values = [ Asset_value (uuid 70) ] + ; values_truncated = false + } + in + observe r (raw_block ~properties:[ property ] 1 900); + loaded r q (G.Uuid.to_string (uuid 1)) []; + change r [ 70 ]; + check (list_reads q = 1) "typed asset property establishes dependency" ) + ; ( "removed foreign asset retains dependency until membership rebuild" + , fun () -> + let r, q, v, _ = fixture () in + observe r (raw_block ~refs:[ uuid 70 ] 1 900); + loaded r q (G.Uuid.to_string (uuid 1)) [ media_asset 70 ]; + change r [ 70 ]; + check (list_reads q = 1) "asset deletion reads its holder"; + reply r (take_metadata q) []; + check + ((Hashtbl.find v (G.Uuid.to_string (uuid 1))).items = []) + "deleted membership disappears" ) + ; ( "known ancestor liveness change selects descendants" + , fun () -> + let r, q, _, _ = fixture () in + observe r (raw_block 60 900); + observe r (raw_block 1 60); + loaded r q (G.Uuid.to_string (uuid 1)) []; + seed r q 2 []; + change r [ 60 ]; + reply_missing r q 60; + check (list_reads q = 1) "ancestor tombstone refreshes its dependent only" ) + ; ( "unknown ancestor is filled through bounded get_block" + , fun () -> + let r, q, _, _ = fixture () in + observe r (raw_block 1 60); + loaded r q (G.Uuid.to_string (uuid 1)) []; + (* Drain only dependency lookups, never manufacture a list-assets answer. *) + let answer () = + let ticket, request = Queue.take q in + match ticket, request with + | ( Some ticket + , S.Graph_request { request_id; command = P.V2_get_block { block; _ }; _ } ) + when block = uuid 60 -> + let value = + match block_response (raw_block 60 61) with + | P.V2_response { outcome; _ } -> outcome + in + R.receive + r + ticket + (S.Graph_response + (P.V2_response { api_version = 2; request_id; outcome = value })) + | _ -> failwith "unknown ancestor requires a public bounded get_block" + in + if Queue.is_empty q then change r [ 61 ]; + answer (); + reply_missing r q 61; + check (list_reads q = 1) "newly discovered ancestor selects the holder" ) + ; ( "membership move and parent structure select existing root" + , fun () -> + let r, q, _, _ = fixture () in + seed r q 1 []; + observe r (raw_block 1 60); + R.changes + r + [ window + ~structures: + [ P.V2_children_interest (uuid 900); V2_children_interest (uuid 60) ] + [ 1 ] + ]; + check (list_reads q = 1) "move rebuilds one matching root" ) + ; ( "changes coalesce while membership request is pending" + , fun () -> + let r, q, _, _ = fixture () in + seed r q 1 []; + membership_change r 1 70; + let old = take_metadata q in + for n = 1 to 32 do + membership_change r 1 (70 + n) + done; + check (list_reads q = 0) "pending membership request cannot duplicate"; + reply r old []; + check (list_reads q = 1) "merged changed membership has one followup" ) + ; ( "queued hidden query is retired before admission" + , fun () -> + let r, q, _, a = fixture () in + a := false; + for n = 1 to 64 do + R.root_visible r ~root:(G.Uuid.to_string (uuid n)) true + done; + for n = 1 to 64 do + R.root_visible r ~root:(G.Uuid.to_string (uuid n)) false + done; + a := true; + R.pump r; + check (list_reads q = 0) "last owner hide retires requests never admitted"; + R.root_visible r ~root:(G.Uuid.to_string (uuid 1)) true; + check (list_reads q = 1) "reactivation starts exactly one first page" ) + ; ( "metadata failure requires explicit Retry" + , fun () -> + let r, q, v, _ = fixture () in + seed r q 1 []; + membership_change r 1 70; + let ticket, _, _ = take_metadata q in + R.reject r ticket; + let root = G.Uuid.to_string (uuid 1) in + R.root_visible r ~root true; + R.pump r; + check + (list_reads q = 0 && Option.is_some (Hashtbl.find v root).error) + "failed membership stays failed without automatic retry"; + R.retry r ~root ~asset:""; + check (list_reads q = 1) "explicit Retry starts one metadata read" ) + ; ( "scope reset rejects late membership" + , fun () -> + let r, q, v, _ = fixture () in + seed r q 1 []; + membership_change r 1 70; + let old = take_metadata q in + R.reset r ~graph_generation:(Some 2); + R.root_visible r ~root:(G.Uuid.to_string (uuid 1)) true; + let fresh = take_metadata q in + reply r old [ media_asset 2 ]; + reply r fresh [ media_asset 3 ]; + check + ((List.hd (Hashtbl.find v (G.Uuid.to_string (uuid 1))).items).asset.uuid + = uuid 3) + "scope reset rejects old membership" ) + ; ( "successful File survives descriptor and resync" + , fun () -> + let r, q, v, _ = fixture () in + let descriptor c = + A.create + ~uuid:(uuid 2) + ~source: + (Managed + (Some + (A.version ~checksum:(String.make 64 c) ~file_type:"png" + |> Result.get_ok))) + ~current_checksum:None + ~size:None + ~dimensions:None + |> Result.get_ok + in + let asset = descriptor 'a' + and root = G.Uuid.to_string (uuid 1) in + seed r q 1 [ asset ]; + let token = (List.hd (Hashtbl.find v root).items).token in + R.asset_visible r ~root ~asset:token true; + Queue.clear q; + let scope : S.asset_scope = + { account = + { managed_sync_origin = Uri.of_string "https://sync.example" + ; user_id = "u" + ; account_generation = 1 + ; presentation_generation = 1 + ; lifecycle_generation = 1L + } + ; graph_id = uuid 900 + ; graph_generation = 1 + } + in + R.notice + r + scope + (Asset_availability + { consumer = token; asset = asset.uuid; availability = Ready "cached" }); + let ticket, _ = Queue.take q in + R.receive r (Option.get ticket) (Asset_file (Some ("held", "/cache/held"))); + observe r (raw_block ~refs:[ asset.uuid ] 1 900); + change r [ 2 ]; + check + (Queue.is_empty q) + "successful asset UUID notification performs no metadata or point read"; + change r [ 1 ]; + reply r (take_metadata q) [ descriptor 'b' ]; + let item = List.hd (Hashtbl.find v root).items in + check + (item.token = token + && item.presentation = Journal_media.File "/cache/held" + && item.asset = descriptor 'b') + "same UUID retains successful lease while publishing updated metadata"; + check (file_commands q = 0) "metadata cannot reacquire/drop the successful File"; + R.resync r; + check (list_reads q = 1) "real resync rebuilds membership"; + reply r (take_metadata q) [ descriptor 'c' ]; + check + ((List.hd (Hashtbl.find v root).items).presentation + = Journal_media.File "/cache/held") + "resync preserves successful same-UUID File"; + check (file_commands q = 0) "resync cannot reacquire/drop a same-UUID lease"; + observe r (raw_block ~refs:[] 1 900); + change r [ 1 ]; + reply r (take_metadata q) []; + check + ((Hashtbl.find v root).items = []) + "holder ref removal removes successful membership"; + check + (Queue.to_seq q + |> Seq.exists (fun (_, request) -> + match request with + | S.Release_asset_file { handle = "held"; _ } -> true + | _ -> false)) + "real membership removal releases the retained File lease" ) + ] ;; diff --git a/test/journal_media_test.ml b/test/journal_media_test.ml index b463802..8a54171 100644 --- a/test/journal_media_test.ml +++ b/test/journal_media_test.ml @@ -101,6 +101,82 @@ let test_external_urls () = ] ;; +let () = + let failed = ref 0 in + let run name test = + try + test (); + Printf.printf "PASS %s\n%!" name + with + | exn -> + incr failed; + Printf.printf "FAIL %s: %s\n%!" name (Printexc.to_string exn) + in + run "loading cannot Retry" (fun () -> + let loading, _ = P.step P.empty (show (asset 'a')) in + let _, effects = P.step loading Retry_requested in + check (effects = []) "loading ignores Retry instead of starting another download"); + run "same UUID descriptor preserves successful File" (fun () -> + let loading, _ = P.step P.empty (show (asset 'a')) in + let pending, effects = ready loading "cached" in + let visible, _ = + P.step pending (Acquired (acquire effects, Some ("held", "/cache/held"))) + in + let updated, effects = P.step visible (show (asset 'b')) in + check + (effects = [] && P.presentation updated = File "/cache/held") + "same UUID descriptor update neither releases nor reacquires a successful File"); + run "failed Retry is one explicit attempt" (fun () -> + let loading, _ = P.step P.empty (show (asset 'a')) in + let failed, _ = + P.step + loading + (Availability + { scope + ; consumer = "visible" + ; availability = + Failed { failure = Network; attempts = 1; retry_scheduled = false } + }) + in + let retrying, effects = P.step failed Retry_requested in + check (List.length effects = 1) "failed Retry starts one attempt"; + let _, duplicate = P.step retrying Retry_requested in + check (duplicate = []) "repeated Retry while loading starts no additional attempt"); + run "successful File ignores late statuses and Retry" (fun () -> + let loading, _ = P.step P.empty (show (asset 'a')) in + let pending, effects = ready loading "cached" in + let visible, _ = + P.step pending (Acquired (acquire effects, Some ("held", "/cache/held"))) + in + List.iter + (fun availability -> + let stable, effects = + P.step visible (Availability { scope; consumer = "visible"; availability }) + in + check + (effects = [] && P.presentation stable = File "/cache/held") + "successful File is stable across transfer notifications") + [ S.Asset.Queued + ; Downloading + ; Ready "other" + ; Failed { failure = Network; attempts = 1; retry_scheduled = false } + ]; + let _, effects = P.step visible Retry_requested in + check (effects = []) "successful File cannot Retry"); + run "offscreen releases then local acquisition is allowed" (fun () -> + let loading, _ = P.step P.empty (show (asset 'a')) in + let pending, effects = ready loading "cached" in + let visible, _ = + P.step pending (Acquired (acquire effects, Some ("held", "/cache/held"))) + in + let hidden, effects = P.step visible Hide in + check (List.length (releases effects) = 1) "Hide releases exactly one lease"; + let shown, _ = P.step hidden (show (asset 'b')) in + let _, effects = ready shown "cached" in + check (List.length effects = 1) "reappearance can acquire the local file once"); + check (!failed = 0) (Printf.sprintf "%d media retry cases failed" !failed) +;; + let () = test_external_urls (); let state, effects = P.step P.empty (show (asset 'a')) in @@ -153,12 +229,10 @@ let () = (P.presentation hidden = Hidden && List.length (releases effects) = 1) "late acquisition is released without display"; let replacement, effects = P.step visible (show (asset 'b')) in - check (List.length (releases effects) = 1) "version replacement releases old file"; + check (effects = []) "same UUID metadata update preserves the displayed lease"; check - (match P.presentation replacement with - | Placeholder _ -> true - | _ -> false) - "new version never displays old bytes"; + (P.presentation replacement = File "/cache/a.png") + "DB descriptor updates preserve an already successful local file"; let unchanged, effects = P.step replacement diff --git a/test/journal_routes_test.ml b/test/journal_routes_test.ml index 98c32d8..d35452a 100644 --- a/test/journal_routes_test.ml +++ b/test/journal_routes_test.ml @@ -2324,8 +2324,237 @@ let test_typed_detached_duplicate_pending_mutations () = "runtime reload lost the retained independent attempt" ;; +let multiselect_pick index path source_identity = + let identity offset = + Printf.sprintf "71000000-0000-4000-a000-%012d" ((index * 4) + offset) + in + `Assoc + [ "operation", `String (identity 0) + ; "asset", `String (identity 1) + ; "localMutation", `String (identity 2) + ; "metadataMutation", `String (identity 3) + ; "path", `String path + ; "title", `String (Filename.basename path) + ; "type", `String "png" + ; "sourceIdentity", `String source_identity + ] +;; + +let multiselect_staged index path source_identity = + match + Journal_asset_import.decode_event + (Yojson.Basic.to_string (multiselect_pick index path source_identity)) + with + | Ok (Picked (staged, _)) -> staged + | _ -> fail "valid legacy selection did not decode" +;; + +let with_multiselect_files count f = + let files = List.init count (fun _ -> Filename.temp_file "journal-import-" ".png") in + Fun.protect + ~finally:(fun () -> + List.iter (fun path -> if Sys.file_exists path then Sys.remove path) files) + (fun () -> f files) +;; + +let test_multiselect_batch_boundary () = + List.iter + (fun (items, error) -> + let payload = + Yojson.Basic.to_string + (`Assoc + [ "action", `String "picked-batch" + ; "items", `List items + ; "request", `Assoc [ "id", `Int 1 ] + ; ( "error" + , match error with + | None -> `Null + | Some message -> `String message ) + ]) + in + match Journal_asset_import.decode_event payload with + | Ok (Picked_batch (decoded, Some 1, actual_error)) -> + require + (List.map Journal_asset_import.staged_path decoded + = List.map + (fun item -> + Yojson.Basic.Util.member "path" item |> Yojson.Basic.Util.to_string) + items) + "batch reordered valid siblings"; + require (actual_error = error) "batch lost its partial failure feedback" + | _ -> fail "ordered/empty/partial-failure native batch was rejected") + [ ( [ multiselect_pick 0 "/tmp/a.png" "files:a" + ; multiselect_pick 1 "/tmp/b.png" "files:b" + ] + , None ) + ; [], None + ; [ multiselect_pick 1 "/tmp/b.png" "photos:b" ], Some "One photo could not be read." + ; [], Some "Unable to read the selected photos." + ]; + require + (Result.is_error + (Journal_asset_import.decode_event + {|{"action":"picked-batch","items":[{}],"request":{"id":1}}|})) + "malformed batch item acquired ownership" +;; + +let test_multiselect_order_duplicates_and_readd () = + let module R = Application.Root_navigation in + with_multiselect_files 4 (function + | [ a; b; c; duplicate ] -> + let picks = + List.mapi + (fun i path -> multiselect_staged i path ("files:" ^ string_of_int i)) + [ a; b; c ] + in + let start = + R.create ~graph_generation:1 + |> fun s -> + R.step s Capture_opened |> fun s -> R.step s (Capture_picker_requested Files) + in + let selected = R.step start (Capture_assets_picked (picks, Some 1, None)) in + let current s = Journal_capture.pending_attachments (Option.get (R.capture s)) in + require + (List.map Journal_asset_import.staged_path (current selected) = [ a; b; c ]) + "selection order was lost"; + let duplicate_pick = multiselect_staged 4 duplicate "files:1" in + let deduped = R.step selected (Capture_asset_picked (duplicate_pick, Some 1)) in + require (current deduped = picks) "same source added a duplicate attachment"; + require (not (Sys.file_exists duplicate)) "duplicate staging copy was leaked"; + let repeated = R.step deduped (Capture_asset_picked (List.hd picks, Some 1)) in + require + (current repeated = picks && Sys.file_exists a) + "replayed event removed an owned source or added a duplicate"; + let removed = + Journal_capture.remove_attachment + (Option.get (R.capture repeated)) + ~token:(Journal_asset_import.staged_token (List.nth picks 1)) + in + let readded = Journal_capture.add_attachment removed (List.nth picks 1) in + require + (List.map + Journal_asset_import.staged_path + (Journal_capture.pending_attachments readded) + = [ a; c; b ]) + "removed source could not be readded at selection tail" + | _ -> assert false) +;; + +let test_multiselect_rejected_resource_ownership () = + let module R = Application.Root_navigation in + with_multiselect_files (Journal_capture.attachment_limit + 2) (fun files -> + let picks = + List.mapi + (fun i path -> multiselect_staged i path ("files:" ^ string_of_int i)) + files + in + let start = + R.create ~graph_generation:1 + |> fun s -> + R.step s Capture_opened |> fun s -> R.step s (Capture_picker_requested Files) + in + let full = + List.fold_left + (fun s p -> R.step s (Capture_asset_picked (p, Some 1))) + start + (List.filteri (fun i _ -> i < Journal_capture.attachment_limit) picks) + in + let overflow = List.nth picks Journal_capture.attachment_limit in + let next = R.step full (Capture_asset_picked (overflow, Some 1)) in + require + (List.length (Journal_capture.pending_attachments (Option.get (R.capture next))) + = Journal_capture.attachment_limit) + "capacity changed"; + require + (not (Sys.file_exists (Journal_asset_import.staged_path overflow))) + "capacity rejection leaked its staging copy"; + let saving = + Journal_capture.update_source (Option.get (R.capture start)) ~source:"Saving owner" + |> fun capture -> + fst + (Journal_capture.admit_save + capture + ~mutation_id:"71000000-0000-4000-9000-000000000090" + ~block_id:"71000000-0000-4000-a000-000000000090" + ~sibling_order:"z" + ~calendar_generation:1L + ~creation_time:(creation_time 1)) + in + let rejected = List.nth picks (Journal_capture.attachment_limit + 1) in + let after = + R.step + (R.step start (Capture_admitted saving)) + (Capture_asset_picked (rejected, Some 1)) + in + require (R.capture after = Some saving) "saving capture accepted an async attachment"; + require + (not (Sys.file_exists (Journal_asset_import.staged_path rejected))) + "saving rejection leaked its staging copy") +;; + +let test_multiselect_graph_owner_and_cancel () = + let module R = Application.Root_navigation in + let graph index = + Logseq_db_types.Graph_types.Uuid.of_string + (Printf.sprintf "71000000-0000-4000-b000-%012d" index) + |> Result.get_ok + in + with_multiselect_files 2 (function + | [ kept; late ] -> + let start = + R.create ~graph_generation:1 + |> fun s -> + R.step s (Graph_replaced { graph_id = Some (graph 1); generation = 1 }) + |> fun s -> + R.step s Capture_opened |> fun s -> R.step s (Capture_picker_requested Photos) + in + let pick = multiselect_staged 1 kept "photos:kept" in + let selected = R.step start (Capture_asset_picked (pick, Some 1)) in + require + (Journal_asset_import.is_dismissal {|{"action":"dismissed","request":1}|}) + "picker cancellation did not decode"; + let collapsed = R.step selected Capture_closed in + require + (R.capture collapsed = R.capture selected && Sys.file_exists kept) + "cancellation/collapse erased owned attachments"; + let other = + R.step collapsed (Graph_replaced { graph_id = Some (graph 2); generation = 2 }) + |> fun s -> R.step s Capture_opened + in + require + (Sys.file_exists kept) + "graph retention deleted a retained draft's attachment"; + let stale = + R.step + other + (Capture_asset_picked (multiselect_staged 2 late "photos:late", Some 1)) + in + require + (Journal_capture.pending_attachments (Option.get (R.capture stale)) = []) + "old graph completion acquired a new draft"; + require (not (Sys.file_exists late)) "old graph completion leaked its staging copy"; + let returned = + R.step stale (Graph_replaced { graph_id = Some (graph 1); generation = 3 }) + in + require + (Journal_capture.pending_attachments (Option.get (R.capture returned)) = [ pick ] + && Sys.file_exists kept) + "returning graph lost staged ownership"; + let other = + R.step returned (Graph_replaced { graph_id = Some (graph 2); generation = 4 }) + in + ignore (R.step other Account_cleared); + require (not (Sys.file_exists kept)) "sign-out leaked retained graph staging" + | _ -> assert false) +;; + let tests = [ "typed path prefix and loading owners", test_typed_path_prefix_and_loading_owners + ; "multiselect batch boundary", test_multiselect_batch_boundary + ; "multiselect order duplicates and readd", test_multiselect_order_duplicates_and_readd + ; ( "multiselect rejected resource ownership" + , test_multiselect_rejected_resource_ownership ) + ; "multiselect graph owner and cancel", test_multiselect_graph_owner_and_cancel ; "typed covered failure and branch", test_typed_covered_failure_missing_and_branch ; "typed shared data and aggregate Undo", test_typed_shared_data_delete_undo_and_drafts ; "typed duplicate detached mutations", test_typed_detached_duplicate_pending_mutations diff --git a/test/journal_semantics_test.ml b/test/journal_semantics_test.ml index af53e22..77bb8a1 100644 --- a/test/journal_semantics_test.ml +++ b/test/journal_semantics_test.ml @@ -45,6 +45,49 @@ let block ;; module V = Ui.View +module Wire_nodes = Set.Make (Int) + +let track_wire_teardown on_drop = + let parents = Hashtbl.create 64 in + let children = Hashtbl.create 64 in + let children_of node = + Option.value (Hashtbl.find_opt children node) ~default:Wire_nodes.empty + in + let unlink node = + match Hashtbl.find_opt parents node with + | None -> () + | Some parent -> + Hashtbl.replace children parent (Wire_nodes.remove node (children_of parent)); + Hashtbl.remove parents node + in + let drop node = + unlink node; + Wire_nodes.iter (Hashtbl.remove parents) (children_of node); + Hashtbl.remove children node; + on_drop node + in + let rec detach node = + Wire_nodes.iter detach (children_of node); + drop node + in + function + | Lui_protocol.InsertChild (parent, child, _) | MoveChild (parent, child, _) -> + unlink child; + Hashtbl.replace parents child parent; + Hashtbl.replace children parent (Wire_nodes.add child (children_of parent)) + | RemoveChild (parent, child) -> + if Hashtbl.find_opt parents child = Some parent then unlink child + | DropNode node -> drop node + | DetachSubtree node -> detach node + | _ -> () +;; + +let dropped_wire_nodes ops = + let dropped = Hashtbl.create 64 in + let track = track_wire_teardown (fun node -> Hashtbl.replace dropped node ()) in + List.iter track ops; + dropped +;; let test_timeline_media_targets () = let targets = ref [] in @@ -925,7 +968,7 @@ let test_file_cards_use_actual_metadata () = let test_attachment_unavailable_keeps_retry () = with_mounted - (media_view [ media_item 1 "jpg" (Placeholder "Not downloaded") None ]) + (media_view [ media_item 1 "jpg" (Failed "Not downloaded") None ]) (fun _ ops -> require (has_text (ops ()) "Not downloaded" && has_text (ops ()) "Retry") @@ -1374,6 +1417,7 @@ let check_stable_image_slots ~entry ~images ~width ~height = images; let check () = let operations = ops () in + let dropped = dropped_wire_nodes operations in List.iter (fun (_, node) -> let prop property = @@ -1391,12 +1435,7 @@ let check_stable_image_slots ~entry ~images ~width ~height = && prop HeightValue = Some (IntValue height)) "image availability changed the final slot dimensions"; require - (not - (List.exists - (function - | Lui_protocol.DropNode id -> id = node - | _ -> false) - operations)) + (not (Hashtbl.mem dropped node)) "image availability replaced a reserved slot") initial in @@ -1552,14 +1591,16 @@ let test_native_list_payload_binds_nested_contents () = transfer requests and leases remain covered by their state owners. *) let mounted_nodes ops property expected = let values = Hashtbl.create 64 in + let track_teardown = track_wire_teardown (Hashtbl.remove values) in List.iter - (function - | Lui_protocol.SetProp (node, key, value) when key = property -> - Hashtbl.replace values node value - | Lui_protocol.RemoveProp (node, key) when key = property -> - Hashtbl.remove values node - | Lui_protocol.DropNode node -> Hashtbl.remove values node - | _ -> ()) + (fun op -> + track_teardown op; + match op with + | Lui_protocol.SetProp (node, key, value) when key = property -> + Hashtbl.replace values node value + | Lui_protocol.RemoveProp (node, key) when key = property -> + Hashtbl.remove values node + | _ -> ()) ops; Hashtbl.fold (fun node value nodes -> if value = expected then node :: nodes else nodes) @@ -1577,16 +1618,12 @@ let current_text ops text = mounted_nodes ops Lui_protocol.TextValue (Lui_protocol.StringValue text) <> [] ;; -let require_nodes_retained nodes delta = +let require_nodes_retained nodes operations = + let dropped = dropped_wire_nodes operations in List.iter (fun node -> require - (not - (List.exists - (function - | Lui_protocol.DropNode id -> id = node - | _ -> false) - delta)) + (not (Hashtbl.mem dropped node)) "item presentation dropped an unaffected body, slot, or sibling node") nodes ;; @@ -1628,14 +1665,12 @@ let test_reactive_child_gallery_routes_and_preserves_siblings () = assert_event (asset_id first) first.token; List.iter (fun presentation -> - let before = List.length (ops ()) in Store.update store ~root:(asset_id first) (Some (media_state [ { first with presentation } ])); ignore (Lui_app.flush app); - let delta = List.filteri (fun index _ -> index >= before) (ops ()) in - require_nodes_retained [ first_slot; second_slot; body; sibling ] delta; + require_nodes_retained [ first_slot; second_slot; body; sibling ] (ops ()); require (node AccessibilityIdentifier ("journal-image-slot:" ^ asset_id first) = first_slot) @@ -1727,7 +1762,6 @@ let test_reactive_nonimage_structure_and_presentation () = Store.update store ~root:block_id view; ignore (Lui_app.flush app) in - let before = List.length (ops ()) in update (Some (media_state [ { pdf with presentation = File "/tmp/reactive-plan.pdf" } ])); require @@ -1736,13 +1770,11 @@ let test_reactive_nonimage_structure_and_presentation () = require (current_text (ops ()) "PDF · 32.5 KB") "PDF card lost actual descriptor metadata"; - require_nodes_retained - [ body; sibling ] - (List.filteri (fun index _ -> index >= before) (ops ())); + require_nodes_retained [ body; sibling ] (ops ()); update (Some (media_state - [ { pdf with presentation = Placeholder "Unable to open file" } ])); + [ { pdf with presentation = Failed "Unable to open file" } ])); require (current_text (ops ()) "Unable to open file" && current_text (ops ()) "Retry") "nonimage failure lost its current message or retry action"; diff --git a/test/source_boundary_test.ml b/test/source_boundary_test.ml index 54229e5..6ae07e7 100644 --- a/test/source_boundary_test.ml +++ b/test/source_boundary_test.ml @@ -1122,10 +1122,10 @@ let () = "logseq_journal.opam" ~package:"ocaml-signal" ~version:"0.1.0"; - (* Journal's host-code mapping uses the validated FlutterHost protocol; - keep both developer and CI manifests on that compatible revision. *) + (* Journal uses the upstream native host ABI decoder; keep the OCaml + installation and Apple package on the same validated revision. *) let compatible_lui_revision = - "git+https://github.com/logseq/lui.git#adbdf63fe940157824f29262095bb194ebd21404" + "git+https://github.com/logseq/lui.git#91aecb52a1cba2faaf23aac1d64a0bd1cb6549e7" in List.iter (fun relative -> require_occurrences root relative compatible_lui_revision 1) @@ -1156,7 +1156,7 @@ let () = ] in let current_datascript_revision = - "datascript-ocaml.git#b346cdd36e9d01af381da022723d3307fd26760d" + "datascript-ocaml.git#0561660e4894faee250d551ab2a32a6b5c25a5fb" in List.iter (fun relative -> @@ -1570,8 +1570,8 @@ let () = [ "sign_out_request"; "is_prepare_to_terminate_event"; "termination_ready_request" ]; require_text root - "swift/JournalAmplifySession.swift" - [ "Amplify.Auth.signOut()"; "Amplify.Auth.fetchAuthSession()"; "getCognitoTokens()" ]; + "swift/JournalCognitoSession.swift" + [ "func signOut()"; "func freshIDToken()"; "func currentUserID()" ]; require_text root "swift/JournalNativeServices.swift" diff --git a/tool/build_journal_apple.sh b/tool/build_journal_apple.sh index ffe2cd0..a8cc2b7 100755 --- a/tool/build_journal_apple.sh +++ b/tool/build_journal_apple.sh @@ -143,7 +143,7 @@ if [[ $platform == ios-simulator ]]; then # is killed (error 163) no matter what identity signed them. The sim reads # its entitlements from the __TEXT,__entitlements section instead — embed # them at link time like Xcode does, then sign adhoc. This is what makes - # keychain (Amplify sign-in, localAccount) work on the sim. + # keychain (OAuth sign-in, localAccount) work on the sim. ios_entitlements="$build_dir/ios-sim-entitlements.plist" cp "$entitlements_dir/ios-debug-profile.entitlements" "$ios_entitlements" bundle_id=$(plutil -extract CFBundleIdentifier raw "$info_plist") @@ -165,14 +165,27 @@ product_dir="$swift_dir/.build/$triple/debug" [[ -f $product_dir/JournalApp ]] || product_dir="$swift_dir/.build/debug" app_dir=${app_dir_arg:-$build_dir/LogseqJournal.app} rm -rf "$app_dir" +register_callback() { + python3 - "$1" <<'PYPLIST' +import plistlib, sys +from pathlib import Path +path = Path(sys.argv[1]) +info = plistlib.loads(path.read_bytes()) +entries = info.setdefault('CFBundleURLTypes', []) +if not any('logseqjournal' in entry.get('CFBundleURLSchemes', []) for entry in entries): + entries.append({'CFBundleURLName': 'com.logseq.journal.auth', 'CFBundleURLSchemes': ['logseqjournal']}) +path.write_bytes(plistlib.dumps(info)) +PYPLIST +} if [[ $platform == macos ]]; then mkdir -p "$app_dir/Contents/MacOS" "$app_dir/Contents/Resources" cp "$info_plist" "$app_dir/Contents/Info.plist" + register_callback "$app_dir/Contents/Info.plist" cp "$product_dir/JournalApp" "$app_dir/Contents/MacOS/JournalApp" # keychain-access-groups needs a real team id; without one the group is # invalid and AMFI kills the binary, so drop the key for local builds. With a # team id, sign with the Apple Development identity so the entitlement is - # honored (Amplify/keychain then work on macOS too). + # honored (OAuth/keychain then work on macOS too). macos_entitlements="$build_dir/macos-entitlements.plist" cp "$entitlements_dir/macos-debug-profile.entitlements" "$macos_entitlements" bundle_id=$(plutil -extract CFBundleIdentifier raw "$info_plist") @@ -194,6 +207,7 @@ else # iOS bundles are flat; an empty Contents/ dir breaks install + codesign. mkdir -p "$app_dir" cp "$info_plist" "$app_dir/Info.plist" + register_callback "$app_dir/Info.plist" cp "$product_dir/JournalApp" "$app_dir/JournalApp" # Plain adhoc signature — the sim's entitlements already live in the # __TEXT,__entitlements section embedded at link time above. Do NOT pass diff --git a/tool/test_lui_native_events.py b/tool/test_lui_native_events.py index 8b4613a..adc3686 100644 --- a/tool/test_lui_native_events.py +++ b/tool/test_lui_native_events.py @@ -16,7 +16,8 @@ ROOT = Path(__file__).resolve().parents[1] LUI = Path(os.environ.get("JOURNAL_LUI_PACKAGE_PATH", ROOT / "../lui/platform/apple")).resolve() -SCRATCH = ROOT / "_build/apple-tests/native-events/lui" +SCRATCH = Path(os.environ.get("JOURNAL_LUI_TEST_SCRATCH", + Path(tempfile.gettempdir()) / "journal-native-events-lui")) def run(command, cwd=ROOT): diff --git a/tool/test_swiftui_amplify.py b/tool/test_swiftui_amplify.py deleted file mode 100644 index 905b36d..0000000 --- a/tool/test_swiftui_amplify.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Build the actual Amplify acceptance view through an isolated LUI probe host. - -Stages a disposable host (tool/lui_probe_host.py) with the Amplify swift -sources, apple-tests/amplify/JournalAmplifyAcceptance.swift as the visible -probe view, and apple-tests/amplify/hub_fixture.ml as the embedded OCaml app -(a Lui_app static view self-registered via Journal_bridge.register). Synthetic -empty entitlements keep the probe off the production Keychain access group. -Launch the assembled .app binary and inspect the PASS/FAIL view with native UI -tools. No account lookup, token retrieval or sign-out is requested. -""" -import argparse -import hashlib -import json -import plistlib -from pathlib import Path -import shutil -import sys -import tempfile - -sys.path.insert(0, str(Path(__file__).resolve().parent)) -import lui_probe_host - -parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument('--host', type=Path, - help='Disposable host directory (defaults to a fresh tempdir)') -parser.add_argument('--platform', choices=['macos', 'ios-simulator'], default='macos') -parser.add_argument('--native-object', type=Path, - help='Complete OCaml object embedding hub_fixture.ml') -args = parser.parse_args() -root = lui_probe_host.ROOT -host = (args.host or Path(tempfile.mkdtemp(prefix='journal-amplify-probe-'))).resolve() -if not host.is_relative_to(Path(tempfile.gettempdir()).resolve()): - raise SystemExit('A disposable host is required') -result = {'host': str(host), 'inputs': {}, 'commands': []} - -sources = sorted((root/'swift').glob('*.swift')) + [ - root/'apple-tests/amplify/JournalAmplifyAcceptance.swift'] -for source in sources: - result['inputs'][str(source.relative_to(root))] = hashlib.sha256(source.read_bytes()).hexdigest() -fixture = root/'apple-tests/amplify/hub_fixture.ml' -result['inputs'][str(fixture.relative_to(root))] = hashlib.sha256(fixture.read_bytes()).hexdigest() - -lui_probe_host.stage(host, app_swift='''import SwiftUI -@main struct ApplicationHost: App { - var body: some Scene { - WindowGroup { JournalAmplifyAcceptance().frame(minWidth: 650, minHeight: 180) } - } -} -''', bundle_id='org.logseq.journal.amplify-probe', display_name='Logseq Journal') -shutil.copy2(root/'apple-tests/amplify/JournalAmplifyAcceptance.swift', - host/'swift'/'JournalAmplifyAcceptance.swift') - -# Synthetic identity: the probe needs no production Keychain access group. -entitlements = host/'entitlements' -entitlements.mkdir(exist_ok=True) -for name in ['macos-debug-profile.entitlements', 'ios-debug-profile.entitlements', - 'macos-release.entitlements']: - (entitlements/name).write_bytes(plistlib.dumps({})) - -app = lui_probe_host.build(host, platform=args.platform, - app_name='JournalAmplifyProbe.app', - native_object=args.native_object, - extra_env={'JOURNAL_ENTITLEMENTS_DIR': str(entitlements)}) -result['commands'].append({'app': str(app)}) -(host/'amplify-test-results.json').write_text(json.dumps(result, indent=2)+'\n') -print(json.dumps(result['commands'][-1]), flush=True) diff --git a/tool/test_swiftui_cognito.py b/tool/test_swiftui_cognito.py new file mode 100644 index 0000000..edd3c72 --- /dev/null +++ b/tool/test_swiftui_cognito.py @@ -0,0 +1,14 @@ +"""Production owner with synthetic signed issuer; no real login, Keychain or GUI.""" +from pathlib import Path +import subprocess +import tempfile +root = Path(__file__).resolve().parents[1] +with tempfile.TemporaryDirectory(prefix='journal-cognito-') as directory: + destination = Path(directory) + sources = ['JournalPlatformWire.swift', 'JournalPlatformServices.swift', + 'JournalAuthentication.swift', 'JournalCognitoOAuth.swift', 'JournalCognitoSession.swift'] + subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', '-module-cache-path', + str(destination/'modules'), *[str(root/'swift'/s) for s in sources], + str(root/'apple-tests/authentication/JournalCognitoTests.swift'), + '-o', str(destination/'tests')], check=True) + subprocess.run([str(destination/'tests')], check=True, timeout=60)