diff --git a/doc/rfc/submitqueue/history-api.md b/doc/rfc/submitqueue/history-api.md index 3b2fd533..f8899075 100644 --- a/doc/rfc/submitqueue/history-api.md +++ b/doc/rfc/submitqueue/history-api.md @@ -1,21 +1,21 @@ -# Gateway History APIs +# Gateway Request History APIs -Design notes for gateway history APIs that return retained lifecycle events selected by SubmitQueue request ID or change ID. +Design notes for gateway history APIs that return retained lifecycle events selected by SubmitQueue request ID or change URI. This document captures **design decisions and rationale only**. ## Problem -Users need to inspect how a request progressed through SubmitQueue, not only its current reconciled state. They may start with the `sqid` returned by `Land` or with a provider-specific change ID represented by a change URI supplied to `Land`. The existing `Status` API collapses the append-only request log into one current status, which is appropriate for polling but hides the sequence of events needed for debugging and lifecycle displays. +Users need to inspect how a request progressed through SubmitQueue, not only its current reconciled state. They may start with the `sqid` returned by `Land` or with a provider-specific change URI supplied to `Land`. The request-summary API collapses the append-only request log into one current status, which is appropriate for polling but hides the sequence of events needed for debugging and lifecycle displays. The gateway owns the request log and is the only service that reads it. The history APIs preserve that ownership boundary by serving gateway-owned `RequestLog` records for the request or requests selected by the caller. ## API Shape -The gateway exposes two read-only RPCs because an `sqid` selects one event list while a change ID may select multiple requests: +The gateway exposes two read-only RPCs because an `sqid` selects one event list while a change URI may select multiple requests: ```proto -message HistoryBySQIDRequest { +message GetRequestHistoryByIDRequest { // Globally unique identifier for a request, as returned by Land. string sqid = 1; } @@ -31,14 +31,14 @@ message HistoryEvent { map metadata = 4; } -message HistoryBySQIDResponse { +message GetRequestHistoryByIDResponse { // Retained request-log events ordered by timestamp_ms ascending with a stable tie-breaker. repeated HistoryEvent events = 1; } -message HistoryByChangeIDRequest { - // Provider-specific change identifier represented by a change URI supplied to Land. - string change_id = 1; +message GetRequestHistoryByChangeURIRequest { + // Exact change URI supplied to Land. + string change_uri = 1; } message RequestHistory { @@ -48,22 +48,22 @@ message RequestHistory { repeated HistoryEvent events = 2; } -message HistoryByChangeIDResponse { +message GetRequestHistoryByChangeURIResponse { // Request histories ordered by the numeric sqid counter ascending. repeated RequestHistory histories = 1; } service SubmitQueueGateway { - rpc HistoryBySQID(HistoryBySQIDRequest) returns (HistoryBySQIDResponse) {} - rpc HistoryByChangeID(HistoryByChangeIDRequest) returns (HistoryByChangeIDResponse) {} + rpc GetRequestHistoryByID(GetRequestHistoryByIDRequest) returns (GetRequestHistoryByIDResponse) {} + rpc GetRequestHistoryByChangeURI(GetRequestHistoryByChangeURIRequest) returns (GetRequestHistoryByChangeURIResponse) {} } ``` -`HistoryBySQID` returns one list of events for exactly one request. `HistoryByChangeID` returns a list of request histories because the same change can be submitted more than once. Each history includes its `sqid` so callers can distinguish submissions and use the identifier with other gateway APIs. +`GetRequestHistoryByID` returns one list of events for exactly one request. `GetRequestHistoryByChangeURI` returns a list of request histories because the same change can be submitted more than once. Each history includes its `sqid` so callers can distinguish submissions and use the identifier with other gateway APIs. ## Status Contract -`HistoryEvent.status` is a string, not a protobuf enum. Its value is populated from `entity.RequestStatus`, the same customer-facing status type stored in `RequestLog` and returned by `Status`. +`HistoryEvent.status` is a string, not a protobuf enum. Its value is populated from `entity.RequestStatus`, the same customer-facing status type stored in `RequestLog` and returned by `GetRequestSummaryByID` and `GetRequestSummaryByChangeURI`. Keeping the wire field as a string allows SubmitQueue to add request statuses without requiring clients to adopt a new generated enum before they can read the response. Clients must tolerate status strings they do not recognize. @@ -80,7 +80,7 @@ The history APIs do not introduce a projection table or persist a second history `RequestVersion` is intentionally excluded. It is an internal reconciliation signal used to determine the current state and is not part of the public lifecycle event contract. -`RequestID` is excluded from each event because the request is identified by the `HistoryBySQID` request or by `RequestHistory.sqid`. +`RequestID` is excluded from each event because the request is identified by the `GetRequestHistoryByID` request or by `RequestHistory.sqid`. ## Ordering @@ -90,7 +90,7 @@ Request-log timestamps are generated by callers, not by the storage backend. The Multiple events may have the same timestamp. Events with equal timestamps are ordered by a stable, implementation-defined tie-breaker so repeated reads of the same retained rows return the same sequence. The tie-breaker is not exposed in the API because it has no lifecycle meaning. For example, the MySQL implementation uses the persisted `salt` column as its secondary sort key. -`HistoryByChangeIDResponse.histories` is ordered by the numeric SQID counter ascending. Implementations must parse the counter rather than compare SQIDs lexicographically, so `main/2` precedes `main/10`. +`GetRequestHistoryByChangeURIResponse.histories` is ordered by the numeric SQID counter ascending. Implementations must parse the counter rather than compare SQIDs lexicographically, so `main/2` precedes `main/10`. ## Preserve Every Stored Event @@ -110,16 +110,16 @@ A SubmitQueue request has a bounded lifecycle under normal operation, and a chan The history APIs are eventually consistent. Direct request-log writes become visible after storage persistence, while events sent through the log topic become visible after the gateway consumes and persists them. A successful response may therefore briefly omit recently emitted events. -Unlike `Status`, the history APIs do not reconcile competing log entries. They expose retained event sequences directly. +Unlike request-summary retrieval, the history APIs do not reconcile competing log entries. They expose retained event sequences directly. ## Errors -Error behavior follows the conventions established by `Status`: +Error behavior follows the conventions established by request-summary retrieval: -- An empty `sqid` passed to `HistoryBySQID` is an invalid request. -- An empty `change_id` passed to `HistoryByChangeID` is an invalid request. -- If no request-log records exist for an `sqid`, `HistoryBySQID` returns the existing `RequestNotFoundError`. -- If no retained request histories match a `change_id`, `HistoryByChangeID` returns a change-ID-specific not-found user error. +- An empty `sqid` passed to `GetRequestHistoryByID` is an invalid request. +- An empty `change_uri` passed to `GetRequestHistoryByChangeURI` is an invalid request. +- If no request-log records exist for an `sqid`, `GetRequestHistoryByID` returns the existing `RequestNotFoundError`. +- If no retained request histories match a `change_uri`, `GetRequestHistoryByChangeURI` returns a change-URI-specific not-found user error. - A request-log storage failure is returned as an infrastructure error. Using the existing request not-found error for `sqid` lookups keeps point lookups consistent across the gateway API. @@ -127,7 +127,7 @@ Using the existing request not-found error for `sqid` lookups keeps point lookup ## Flow ```text -HistoryBySQIDRequest(sqid) +GetRequestHistoryByIDRequest(sqid) | v validate sqid @@ -139,12 +139,12 @@ RequestLogStore.List(sqid) project each RequestLog to one HistoryEvent | v -HistoryBySQIDResponse(events) +GetRequestHistoryByIDResponse(events) -HistoryByChangeIDRequest(change_id) +GetRequestHistoryByChangeURIRequest(change_uri) | v -validate change_id +validate change_uri | v resolve matching sqids @@ -156,7 +156,7 @@ RequestLogStore.List(sqid) for each match project each RequestLog to one HistoryEvent | v -HistoryByChangeIDResponse(histories) +GetRequestHistoryByChangeURIResponse(histories) ``` -The API contract does not prescribe how a change ID is mapped to matching requests. That lookup is an implementation concern and must preserve the gateway's ownership of the history read path. +The API contract does not prescribe how a change URI is mapped to matching requests. That lookup is an implementation concern and must preserve the gateway's ownership of the history read path. diff --git a/doc/rfc/submitqueue/status-list-api.md b/doc/rfc/submitqueue/status-list-api.md index e24e218b..720e3e33 100644 --- a/doc/rfc/submitqueue/status-list-api.md +++ b/doc/rfc/submitqueue/status-list-api.md @@ -1,14 +1,14 @@ -# Gateway Status and List APIs +# Gateway Request Summary and List APIs ## RFC Status Proposed. -This RFC replaces the unimplemented design in [list-api.md](list-api.md). It defines the gateway-owned request context and materialized status model used by both `Status` and `List`. +This RFC replaces the unimplemented design in [list-api.md](list-api.md). It defines the gateway-owned request context and materialized status model used by request-summary retrieval and `List`. ## Problem -The gateway currently stores an append-only request log and reconciles it at read time for `Status`. Request-log entries contain status, error, and display metadata, but they do not contain the immutable request context needed by user-facing APIs: queue name, submitted change URIs, and the time the gateway received the request. +The gateway currently stores an append-only request log and reconciles it at read time for request-summary retrieval. Request-log entries contain status, error, and display metadata, but they do not contain the immutable request context needed by user-facing APIs: queue name, submitted change URIs, and the time the gateway received the request. The required read patterns are: @@ -20,16 +20,17 @@ The append-only request log is not shaped for the second or third query. Serving ## Decisions -1. `Status` accepts exactly one selector: sqid or change URI. +1. The gateway exposes `GetRequestSummaryByID` for sqid lookup and `GetRequestSummaryByChangeURI` for exact change URI lookup. 2. A change URI lookup returns all requests containing that exact URI, ordered by receipt time descending. -3. Change URIs are treated as globally meaningful identifiers. `Status` does not require a queue when selecting by URI. +3. Change URIs are treated as globally meaningful identifiers. `GetRequestSummaryByChangeURI` does not require a queue. 4. `List` accepts one queue and a required receipt-time range. It does not accept sqid or URI selectors. 5. The `List` time range is based only on gateway receipt time, not lifecycle overlap. -6. `Status` and `List` return the same materialized current state and immutable request context. Neither endpoint returns the request-log timeline. +6. The request-summary RPCs and `List` return the same materialized current state and immutable request context. Neither endpoint returns the request-log timeline. 7. Materialization happens on write. Reads do not reconcile the append-only log at request time. 8. The data model uses separate query-shaped stores rather than secondary indexes. 9. Existing rows are not backfilled. Requests received before rollout may be unavailable through the new read model. 10. Retention and pruning are outside the scope of this RFC. +11. Land receipts begin in the internal `accepting` state. `accepting` requests are excluded from request-summary retrieval and `List` until successful pipeline publication promotes them to `accepted`, or a later pipeline event proves that publication succeeded. ## Vocabulary @@ -39,13 +40,20 @@ The append-only request log is not shaped for the second or third query. Serving ## API Contract -### Status +### Request Summary Retrieval -`Status` requires exactly one selector: sqid or change URI. An unset selector or an explicitly selected empty value is invalid. +The gateway exposes two explicit RPCs: + +```proto +rpc GetRequestSummaryByID(GetRequestSummaryByIDRequest) returns (GetRequestSummaryByIDResponse) {} +rpc GetRequestSummaryByChangeURI(GetRequestSummaryByChangeURIRequest) returns (GetRequestSummaryByChangeURIResponse) {} +``` + +`GetRequestSummaryByID` requires a non-empty sqid. `GetRequestSummaryByChangeURI` requires a non-empty exact change URI. An sqid lookup returns exactly one request summary when found. A change URI lookup returns every matching request summary ordered by `(received_at_ms DESC, sqid DESC)`. The sqid tie-breaker makes the result deterministic when requests share a millisecond. -Both selector modes return the same response shape: a list of request summaries. Each summary contains sqid, queue, change URIs, receipt time, current status, last error, and display metadata. +`GetRequestSummaryByID` returns one request summary. `GetRequestSummaryByChangeURI` returns a list of request summaries. Each summary contains sqid, queue, change URIs, receipt time, current status, last error, and display metadata. The URI result is intentionally unpaginated and has a hard maximum of 100 requests. A change is expected to have only a small number of SubmitQueue requests, generally fewer than one hundred. Exceeding the maximum returns an error rather than silently truncating the result. @@ -75,7 +83,7 @@ The gateway owns the append-only request log and three new logical read models. The authoritative request summary is keyed by sqid. It contains immutable request context plus the current materialized request-log winner and the reconciliation state needed to compare a later log entry without rereading historical logs. -The immutable context is queue, change URIs, and receipt time. The mutable response state is status, last error, and metadata. Optimistic-lock state is internal to the projection and is not part of the API response. +The immutable context is queue, change URIs, and receipt time. The mutable response state is status, last error, and metadata. The internal `accepting` admission state is not returned by the API. The projection version fields used for optimistic conditional writes are internal and are not part of the API response. The sqid key supports authoritative lookup and conditional status updates for one request without a secondary index. @@ -93,7 +101,7 @@ The URI reverse mapping is logically keyed by `(change_uri, received_at_ms, sqid The mapping repeats `received_at_ms` because receipt time is part of the promised newest-first ordering. This allows the gateway to perform a bounded ordered scan before resolving the matching authoritative summaries. Without receipt time in the mapping, the gateway would have to fetch and sort every request associated with a URI before enforcing the result maximum. -The logical key supports the bounded `Status(change_uri)` newest-first scan and deterministic sqid tie-breaker without fetching every matching summary first. +The logical key supports the bounded `GetRequestSummaryByChangeURI(change_uri)` newest-first scan and deterministic sqid tie-breaker without fetching every matching summary first. The URI is stored in the canonical form received from the validated Land request. URI normalization rules belong to the change contract or source-control integration and are not introduced by this read model. @@ -101,52 +109,58 @@ The URI is stored in the canonical form received from the validated Land request ### Land Receipt -After synchronous validation, Land generates the sqid and one receipt timestamp. The gateway persists the authoritative summary, URI mappings, and queue projection before appending the initial `accepted` request log and before publishing the request to the orchestrator. +After synchronous validation, Land generates the sqid and one receipt timestamp. The gateway persists the authoritative summary in the internal `accepting` state, then publishes the request to the orchestrator. It does not create the URI or queue projections while the request remains `accepting`. + +After publication succeeds, Land appends the initial `accepted` request log. Materializing `accepted`, `started`, or any later event promotes the authoritative summary out of `accepting` and creates the URI and queue projections. This handles the race where the orchestrator emits `started` before Land finishes persisting `accepted`. + +An `accepted` event is the lowest public lifecycle state. A late `accepted` event is retained in request history but must not replace `started` or any later materialized status. -This ordering guarantees that immutable request context exists before the request can produce later status logs or enter the asynchronous pipeline. The logical writes are independent and must be safe to retry for the same sqid and values; conflicting duplicate data is an error. +Pipeline publication is the Land success boundary. If publication fails, Land returns an error and leaves the hidden `accepting` receipt for operational cleanup. If publication succeeds but appending or materializing `accepted` fails, Land still returns the sqid because retrying the RPC would submit a duplicate request that is already in the pipeline. A later pipeline event activates and repairs the public projections. ### Request-Log Materialization Every gateway request-log persistence path uses the same materialization component. It appends the audit log, compares the incoming entry with the authoritative summary, conditionally advances the winner, and propagates the authoritative value to the queue projection. -The winner comparison preserves the existing `Status` behavior: +The winner comparison preserves the existing current-status reconciliation behavior: 1. A terminal request-state entry with a positive request version beats every non-terminal or unversioned winner. 2. Between versioned terminal entries, the greater request version wins. 3. Equal terminal versions use the greater log timestamp as a tie-breaker. 4. When no versioned terminal winner exists, the greater log timestamp wins. +5. Any retained lifecycle event promotes an `accepting` summary into the public projections. +6. `accepted` cannot replace `started` or any later status, even when the accepted log arrives later. Materialization uses optimistic concurrency so stale or out-of-order consumers cannot replace a newer winner. Version arithmetic and reconciliation decisions belong to the materialization component; stores perform only mechanical creates, reads, conditional updates, and bounded page queries. ## Read Flows -### Status by Sqid +### Request Summary by ID -The gateway reads the authoritative summary by sqid and returns its current materialized state and immutable context. It does not fall back to parsing the sqid, reconciling logs, or reading orchestrator stores. +The gateway reads the authoritative summary by sqid and returns its current materialized state and immutable context. An `accepting` summary is treated as not found because it has not crossed the public admission boundary. The gateway does not fall back to parsing the sqid, reconciling logs, or reading orchestrator stores. -### Status by Change URI +### Request Summaries by Change URI -The gateway performs a bounded newest-first scan of the URI reverse mapping and then reads the authoritative summary for each resolved sqid. Results preserve the mapping order. No mapping is a not-found result; exceeding the 100-request maximum is an error. A mapping whose authoritative summary is missing is an internal consistency error that fails the lookup; it is not returned as user-facing not-found and is not silently omitted. +The gateway performs a bounded newest-first scan of the URI reverse mapping and then reads the authoritative summary for each resolved sqid. Results preserve the mapping order. URI mappings are created only when the request reaches `accepted` or a later state, so `accepting` receipts are absent by construction. No mapping is a not-found result; exceeding the 100-request maximum is an error. A mapping whose authoritative summary is missing is an internal consistency error that fails the lookup; it is not returned as user-facing not-found and is not silently omitted. ### List by Queue and Receipt Time -The gateway performs one bounded range scan of the queue projection using queue, receipt-time bounds, and an optional keyset cursor. The ordering key is immutable, so later status updates cannot move an item across an issued cursor. +The gateway performs one bounded range scan of the queue projection using queue, receipt-time bounds, and an optional keyset cursor. Queue projections are created only when the request reaches `accepted` or a later state, so `accepting` receipts are absent by construction. The ordering key is immutable, so later status updates cannot move an item across an issued cursor. ## Consistency The request log remains the append-only audit record. The authoritative summary and queue projection are eventually consistent views of its winning current state. -Because the authoritative summary and queue projection are separate writes, a short interval can exist where `Status` and `List` show different statuses. Retried materialization repairs the queue projection from the authoritative sqid summary until both converge. Neither API reconciles logs during reads to hide this interval. +Because the authoritative summary and queue projection are separate writes, a short interval can exist where request-summary retrieval and `List` show different statuses. Retried materialization repairs the queue projection from the authoritative sqid summary until both converge. Neither API reconciles logs during reads to hide this interval. -Request context has a stronger guarantee than status convergence: it is persisted before the initial log and before the request is published to the orchestrator. Once a queue projection is visible, its queue, sqid, change URIs, and receipt time are complete. +Request context has a stronger guarantee than status convergence: the authoritative receipt is persisted before the request is published to the orchestrator. Public URI and queue projections are activated only by `accepted` or a later event. Once a queue projection is visible, its queue, sqid, change URIs, and receipt time are complete. ## Compatibility and Rollout No existing SQL table requires an in-place breaking modification. The request log remains unchanged, and the new query patterns use additive gateway-owned read models. -There is intentionally no backfill from orchestrator working state or historical request logs. Deployments may create the new stores empty and begin populating them for new Land requests. This creates a behavioral cutoff: requests received before rollout are not guaranteed to resolve through the new `Status` implementation or appear in `List`. +There is intentionally no backfill from orchestrator working state or historical request logs. Deployments may create the new stores empty and begin populating them for new Land requests. This creates a behavioral cutoff: requests received before rollout are not guaranteed to resolve through the new request-summary implementation or appear in `List`. -The Status request change preserves the existing sqid field number but may be source-breaking because generated clients represent the selector as a protobuf `oneof`. Changing Status from singular status fields to a list of request summaries is a breaking response change. The List RPC is additive. +The request-summary RPCs replace the former polymorphic Status shape with explicit ID and change-URI lookups. This is a source-breaking API change. The List RPC is additive. ## Rejected Alternatives @@ -168,4 +182,4 @@ Filtering the existing queue receipt scan would require server-side filtering ou ### Let List Accept Sqid or URI -Sqid lookup and the small URI submission-history lookup belong to `Status`. `List` has one purpose: enumerate queue receipts in a bounded time range. Keeping these contracts separate gives each storage operation one predictable access pattern. +Sqid lookup and the small URI submission-history lookup belong to the request-summary RPCs. `List` has one purpose: enumerate queue receipts in a bounded time range. Keeping these contracts separate gives each storage operation one predictable access pattern. diff --git a/submitqueue/extension/storage/BUILD.bazel b/submitqueue/extension/storage/BUILD.bazel index ccc2bcef..d2081a00 100644 --- a/submitqueue/extension/storage/BUILD.bazel +++ b/submitqueue/extension/storage/BUILD.bazel @@ -8,7 +8,10 @@ go_library( "build_store.go", "change_store.go", "request_log_store.go", + "request_queue_summary_store.go", "request_store.go", + "request_summary_store.go", + "request_uri_store.go", "speculation_path_build_store.go", "speculation_tree_store.go", "storage.go", diff --git a/submitqueue/extension/storage/mock/BUILD.bazel b/submitqueue/extension/storage/mock/BUILD.bazel index 97ceeec5..4b2f4f51 100644 --- a/submitqueue/extension/storage/mock/BUILD.bazel +++ b/submitqueue/extension/storage/mock/BUILD.bazel @@ -8,7 +8,10 @@ go_library( "build_store_mock.go", "change_store_mock.go", "request_log_store_mock.go", + "request_queue_summary_store_mock.go", "request_store_mock.go", + "request_summary_store_mock.go", + "request_uri_store_mock.go", "speculation_path_build_store_mock.go", "speculation_tree_store_mock.go", "storage_mock.go", diff --git a/submitqueue/extension/storage/mock/request_queue_summary_store_mock.go b/submitqueue/extension/storage/mock/request_queue_summary_store_mock.go new file mode 100644 index 00000000..5d7698af --- /dev/null +++ b/submitqueue/extension/storage/mock/request_queue_summary_store_mock.go @@ -0,0 +1,101 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: request_queue_summary_store.go +// +// Generated by this command: +// +// mockgen -source=request_queue_summary_store.go -destination=mock/request_queue_summary_store_mock.go -package=mock +// + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + reflect "reflect" + + entity "github.com/uber/submitqueue/submitqueue/entity" + storage "github.com/uber/submitqueue/submitqueue/extension/storage" + gomock "go.uber.org/mock/gomock" +) + +// MockRequestQueueSummaryStore is a mock of RequestQueueSummaryStore interface. +type MockRequestQueueSummaryStore struct { + ctrl *gomock.Controller + recorder *MockRequestQueueSummaryStoreMockRecorder + isgomock struct{} +} + +// MockRequestQueueSummaryStoreMockRecorder is the mock recorder for MockRequestQueueSummaryStore. +type MockRequestQueueSummaryStoreMockRecorder struct { + mock *MockRequestQueueSummaryStore +} + +// NewMockRequestQueueSummaryStore creates a new mock instance. +func NewMockRequestQueueSummaryStore(ctrl *gomock.Controller) *MockRequestQueueSummaryStore { + mock := &MockRequestQueueSummaryStore{ctrl: ctrl} + mock.recorder = &MockRequestQueueSummaryStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRequestQueueSummaryStore) EXPECT() *MockRequestQueueSummaryStoreMockRecorder { + return m.recorder +} + +// Create mocks base method. +func (m *MockRequestQueueSummaryStore) Create(ctx context.Context, summary entity.RequestQueueSummary) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", ctx, summary) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create. +func (mr *MockRequestQueueSummaryStoreMockRecorder) Create(ctx, summary any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockRequestQueueSummaryStore)(nil).Create), ctx, summary) +} + +// Get mocks base method. +func (m *MockRequestQueueSummaryStore) Get(ctx context.Context, queue string, receivedAtMs int64, requestID string) (entity.RequestQueueSummary, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, queue, receivedAtMs, requestID) + ret0, _ := ret[0].(entity.RequestQueueSummary) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockRequestQueueSummaryStoreMockRecorder) Get(ctx, queue, receivedAtMs, requestID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockRequestQueueSummaryStore)(nil).Get), ctx, queue, receivedAtMs, requestID) +} + +// List mocks base method. +func (m *MockRequestQueueSummaryStore) List(ctx context.Context, query storage.RequestQueueSummaryQuery) ([]entity.RequestQueueSummary, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", ctx, query) + ret0, _ := ret[0].([]entity.RequestQueueSummary) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockRequestQueueSummaryStoreMockRecorder) List(ctx, query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockRequestQueueSummaryStore)(nil).List), ctx, query) +} + +// Update mocks base method. +func (m *MockRequestQueueSummaryStore) Update(ctx context.Context, summary entity.RequestQueueSummary, oldVersion, newVersion int32) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", ctx, summary, oldVersion, newVersion) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update. +func (mr *MockRequestQueueSummaryStoreMockRecorder) Update(ctx, summary, oldVersion, newVersion any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockRequestQueueSummaryStore)(nil).Update), ctx, summary, oldVersion, newVersion) +} diff --git a/submitqueue/extension/storage/mock/request_summary_store_mock.go b/submitqueue/extension/storage/mock/request_summary_store_mock.go new file mode 100644 index 00000000..aeb4adb3 --- /dev/null +++ b/submitqueue/extension/storage/mock/request_summary_store_mock.go @@ -0,0 +1,85 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: request_summary_store.go +// +// Generated by this command: +// +// mockgen -source=request_summary_store.go -destination=mock/request_summary_store_mock.go -package=mock +// + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + reflect "reflect" + + entity "github.com/uber/submitqueue/submitqueue/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockRequestSummaryStore is a mock of RequestSummaryStore interface. +type MockRequestSummaryStore struct { + ctrl *gomock.Controller + recorder *MockRequestSummaryStoreMockRecorder + isgomock struct{} +} + +// MockRequestSummaryStoreMockRecorder is the mock recorder for MockRequestSummaryStore. +type MockRequestSummaryStoreMockRecorder struct { + mock *MockRequestSummaryStore +} + +// NewMockRequestSummaryStore creates a new mock instance. +func NewMockRequestSummaryStore(ctrl *gomock.Controller) *MockRequestSummaryStore { + mock := &MockRequestSummaryStore{ctrl: ctrl} + mock.recorder = &MockRequestSummaryStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRequestSummaryStore) EXPECT() *MockRequestSummaryStoreMockRecorder { + return m.recorder +} + +// Create mocks base method. +func (m *MockRequestSummaryStore) Create(ctx context.Context, summary entity.RequestSummary) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", ctx, summary) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create. +func (mr *MockRequestSummaryStoreMockRecorder) Create(ctx, summary any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockRequestSummaryStore)(nil).Create), ctx, summary) +} + +// Get mocks base method. +func (m *MockRequestSummaryStore) Get(ctx context.Context, requestID string) (entity.RequestSummary, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, requestID) + ret0, _ := ret[0].(entity.RequestSummary) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockRequestSummaryStoreMockRecorder) Get(ctx, requestID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockRequestSummaryStore)(nil).Get), ctx, requestID) +} + +// Update mocks base method. +func (m *MockRequestSummaryStore) Update(ctx context.Context, summary entity.RequestSummary, oldVersion, newVersion int32) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", ctx, summary, oldVersion, newVersion) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update. +func (mr *MockRequestSummaryStoreMockRecorder) Update(ctx, summary, oldVersion, newVersion any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockRequestSummaryStore)(nil).Update), ctx, summary, oldVersion, newVersion) +} diff --git a/submitqueue/extension/storage/mock/request_uri_store_mock.go b/submitqueue/extension/storage/mock/request_uri_store_mock.go new file mode 100644 index 00000000..1904a3f0 --- /dev/null +++ b/submitqueue/extension/storage/mock/request_uri_store_mock.go @@ -0,0 +1,71 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: request_uri_store.go +// +// Generated by this command: +// +// mockgen -source=request_uri_store.go -destination=mock/request_uri_store_mock.go -package=mock +// + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + reflect "reflect" + + entity "github.com/uber/submitqueue/submitqueue/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockRequestURIStore is a mock of RequestURIStore interface. +type MockRequestURIStore struct { + ctrl *gomock.Controller + recorder *MockRequestURIStoreMockRecorder + isgomock struct{} +} + +// MockRequestURIStoreMockRecorder is the mock recorder for MockRequestURIStore. +type MockRequestURIStoreMockRecorder struct { + mock *MockRequestURIStore +} + +// NewMockRequestURIStore creates a new mock instance. +func NewMockRequestURIStore(ctrl *gomock.Controller) *MockRequestURIStore { + mock := &MockRequestURIStore{ctrl: ctrl} + mock.recorder = &MockRequestURIStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRequestURIStore) EXPECT() *MockRequestURIStoreMockRecorder { + return m.recorder +} + +// Create mocks base method. +func (m *MockRequestURIStore) Create(ctx context.Context, mapping entity.RequestURI) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", ctx, mapping) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create. +func (mr *MockRequestURIStoreMockRecorder) Create(ctx, mapping any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockRequestURIStore)(nil).Create), ctx, mapping) +} + +// ListByURI mocks base method. +func (m *MockRequestURIStore) ListByURI(ctx context.Context, changeURI string, limit int) ([]entity.RequestURI, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListByURI", ctx, changeURI, limit) + ret0, _ := ret[0].([]entity.RequestURI) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListByURI indicates an expected call of ListByURI. +func (mr *MockRequestURIStoreMockRecorder) ListByURI(ctx, changeURI, limit any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListByURI", reflect.TypeOf((*MockRequestURIStore)(nil).ListByURI), ctx, changeURI, limit) +} diff --git a/submitqueue/extension/storage/mock/storage_mock.go b/submitqueue/extension/storage/mock/storage_mock.go index c144bc50..6655716f 100644 --- a/submitqueue/extension/storage/mock/storage_mock.go +++ b/submitqueue/extension/storage/mock/storage_mock.go @@ -124,6 +124,20 @@ func (mr *MockStorageMockRecorder) GetRequestLogStore() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRequestLogStore", reflect.TypeOf((*MockStorage)(nil).GetRequestLogStore)) } +// GetRequestQueueSummaryStore mocks base method. +func (m *MockStorage) GetRequestQueueSummaryStore() storage.RequestQueueSummaryStore { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRequestQueueSummaryStore") + ret0, _ := ret[0].(storage.RequestQueueSummaryStore) + return ret0 +} + +// GetRequestQueueSummaryStore indicates an expected call of GetRequestQueueSummaryStore. +func (mr *MockStorageMockRecorder) GetRequestQueueSummaryStore() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRequestQueueSummaryStore", reflect.TypeOf((*MockStorage)(nil).GetRequestQueueSummaryStore)) +} + // GetRequestStore mocks base method. func (m *MockStorage) GetRequestStore() storage.RequestStore { m.ctrl.T.Helper() @@ -152,6 +166,34 @@ func (mr *MockStorageMockRecorder) GetSpeculationPathBuildStore() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSpeculationPathBuildStore", reflect.TypeOf((*MockStorage)(nil).GetSpeculationPathBuildStore)) } +// GetRequestSummaryStore mocks base method. +func (m *MockStorage) GetRequestSummaryStore() storage.RequestSummaryStore { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRequestSummaryStore") + ret0, _ := ret[0].(storage.RequestSummaryStore) + return ret0 +} + +// GetRequestSummaryStore indicates an expected call of GetRequestSummaryStore. +func (mr *MockStorageMockRecorder) GetRequestSummaryStore() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRequestSummaryStore", reflect.TypeOf((*MockStorage)(nil).GetRequestSummaryStore)) +} + +// GetRequestURIStore mocks base method. +func (m *MockStorage) GetRequestURIStore() storage.RequestURIStore { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRequestURIStore") + ret0, _ := ret[0].(storage.RequestURIStore) + return ret0 +} + +// GetRequestURIStore indicates an expected call of GetRequestURIStore. +func (mr *MockStorageMockRecorder) GetRequestURIStore() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRequestURIStore", reflect.TypeOf((*MockStorage)(nil).GetRequestURIStore)) +} + // GetSpeculationTreeStore mocks base method. func (m *MockStorage) GetSpeculationTreeStore() storage.SpeculationTreeStore { m.ctrl.T.Helper() diff --git a/submitqueue/extension/storage/mysql/BUILD.bazel b/submitqueue/extension/storage/mysql/BUILD.bazel index cc04b8c6..cf351b43 100644 --- a/submitqueue/extension/storage/mysql/BUILD.bazel +++ b/submitqueue/extension/storage/mysql/BUILD.bazel @@ -8,7 +8,10 @@ go_library( "build_store.go", "change_store.go", "request_log_store.go", + "request_queue_summary_store.go", "request_store.go", + "request_summary_store.go", + "request_uri_store.go", "speculation_path_build_store.go", "speculation_tree_store.go", "storage.go", diff --git a/submitqueue/extension/storage/mysql/batch_dependent_store.go b/submitqueue/extension/storage/mysql/batch_dependent_store.go index 7bf8047d..64de6431 100644 --- a/submitqueue/extension/storage/mysql/batch_dependent_store.go +++ b/submitqueue/extension/storage/mysql/batch_dependent_store.go @@ -82,7 +82,7 @@ func (s *batchDependentStore) Create(ctx context.Context, batchDependent entity. ) if err != nil { var mysqlErr *mysql.MySQLError - if errors.As(err, &mysqlErr) && mysqlErr.Number == 1062 { + if errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlErrDuplicateEntry { return fmt.Errorf("batch dependent entity batchID=%s: %w", batchDependent.BatchID, storage.ErrAlreadyExists) } return fmt.Errorf("failed to insert batch dependent entity batchID=%s: %w", batchDependent.BatchID, err) diff --git a/submitqueue/extension/storage/mysql/batch_store.go b/submitqueue/extension/storage/mysql/batch_store.go index 4f0168ee..07020dac 100644 --- a/submitqueue/extension/storage/mysql/batch_store.go +++ b/submitqueue/extension/storage/mysql/batch_store.go @@ -93,7 +93,7 @@ func (s *batchStore) Create(ctx context.Context, batch entity.Batch) (retErr err ) if err != nil { var mysqlErr *mysql.MySQLError - if errors.As(err, &mysqlErr) && mysqlErr.Number == 1062 { + if errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlErrDuplicateEntry { return fmt.Errorf("batch entity id=%s: %w", batch.ID, storage.ErrAlreadyExists) } return fmt.Errorf("failed to insert batch entity id=%s: %w", batch.ID, err) diff --git a/submitqueue/extension/storage/mysql/build_store.go b/submitqueue/extension/storage/mysql/build_store.go index b50b8168..cfcf8d99 100644 --- a/submitqueue/extension/storage/mysql/build_store.go +++ b/submitqueue/extension/storage/mysql/build_store.go @@ -71,7 +71,7 @@ func (s *buildStore) Create(ctx context.Context, build entity.Build) (retErr err ) if err != nil { var mysqlErr *mysql.MySQLError - if errors.As(err, &mysqlErr) && mysqlErr.Number == 1062 { + if errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlErrDuplicateEntry { return fmt.Errorf("build entity id=%s: %w", build.ID, storage.ErrAlreadyExists) } return fmt.Errorf("failed to insert build entity id=%s: %w", build.ID, err) diff --git a/submitqueue/extension/storage/mysql/request_queue_summary_store.go b/submitqueue/extension/storage/mysql/request_queue_summary_store.go new file mode 100644 index 00000000..8a292fe0 --- /dev/null +++ b/submitqueue/extension/storage/mysql/request_queue_summary_store.go @@ -0,0 +1,160 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysql + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + + "github.com/go-sql-driver/mysql" + "github.com/uber-go/tally" + + "github.com/uber/submitqueue/platform/metrics" + "github.com/uber/submitqueue/submitqueue/entity" + "github.com/uber/submitqueue/submitqueue/extension/storage" +) + +type requestQueueSummaryStore struct { + db *sql.DB + scope tally.Scope +} + +// NewRequestQueueSummaryStore creates a MySQL-backed RequestQueueSummaryStore. +func NewRequestQueueSummaryStore(db *sql.DB, scope tally.Scope) storage.RequestQueueSummaryStore { + return &requestQueueSummaryStore{db: db, scope: scope} +} + +func (s *requestQueueSummaryStore) Create(ctx context.Context, summary entity.RequestQueueSummary) (retErr error) { + op := metrics.Begin(s.scope, "create") + defer func() { op.Complete(retErr) }() + + changeURIsJSON, metadataJSON, err := marshalSummaryJSON(summary.ChangeURIs, summary.Metadata) + if err != nil { + return fmt.Errorf("failed to marshal queue summary request_id=%s: %w", summary.RequestID, err) + } + _, err = s.db.ExecContext(ctx, ` + INSERT INTO request_summary_by_queue ( + queue, received_at_ms, request_id, change_uris, status, + version, last_error, metadata + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + summary.Queue, summary.ReceivedAtMs, summary.RequestID, changeURIsJSON, + summary.Status, summary.Version, summary.LastError, metadataJSON, + ) + if err != nil { + var mysqlErr *mysql.MySQLError + if errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlErrDuplicateEntry { + return fmt.Errorf("queue summary queue=%s received_at_ms=%d request_id=%s: %w", summary.Queue, summary.ReceivedAtMs, summary.RequestID, storage.ErrAlreadyExists) + } + return fmt.Errorf("failed to insert queue summary request_id=%s: %w", summary.RequestID, err) + } + return nil +} + +func (s *requestQueueSummaryStore) Get(ctx context.Context, queue string, receivedAtMs int64, requestID string) (ret entity.RequestQueueSummary, retErr error) { + op := metrics.Begin(s.scope, "get") + defer func() { op.Complete(retErr) }() + + var changeURIsJSON []byte + var metadataJSON []byte + err := s.db.QueryRowContext(ctx, ` + SELECT queue, received_at_ms, request_id, change_uris, status, + version, last_error, metadata + FROM request_summary_by_queue + WHERE queue = ? AND received_at_ms = ? AND request_id = ?`, queue, receivedAtMs, requestID, + ).Scan(&ret.Queue, &ret.ReceivedAtMs, &ret.RequestID, &changeURIsJSON, &ret.Status, &ret.Version, &ret.LastError, &metadataJSON) + if errors.Is(err, sql.ErrNoRows) { + return entity.RequestQueueSummary{}, storage.WrapNotFound(err) + } + if err != nil { + return entity.RequestQueueSummary{}, fmt.Errorf("failed to get queue summary queue=%s received_at_ms=%d request_id=%s: %w", queue, receivedAtMs, requestID, err) + } + if err := unmarshalSummaryJSON(changeURIsJSON, metadataJSON, &ret.ChangeURIs, &ret.Metadata); err != nil { + return entity.RequestQueueSummary{}, fmt.Errorf("failed to decode queue summary request_id=%s: %w", requestID, err) + } + return ret, nil +} + +func (s *requestQueueSummaryStore) Update(ctx context.Context, summary entity.RequestQueueSummary, oldVersion, newVersion int32) (retErr error) { + op := metrics.Begin(s.scope, "update") + defer func() { op.Complete(retErr) }() + + metadataJSON, err := json.Marshal(normalizeMetadata(summary.Metadata)) + if err != nil { + return fmt.Errorf("failed to marshal queue summary metadata request_id=%s: %w", summary.RequestID, err) + } + result, err := s.db.ExecContext(ctx, ` + UPDATE request_summary_by_queue + SET status = ?, version = ?, last_error = ?, metadata = ? + WHERE queue = ? AND received_at_ms = ? AND request_id = ? AND version = ?`, + summary.Status, newVersion, summary.LastError, metadataJSON, + summary.Queue, summary.ReceivedAtMs, summary.RequestID, oldVersion, + ) + if err != nil { + return fmt.Errorf("failed to update queue summary request_id=%s old_version=%d new_version=%d: %w", summary.RequestID, oldVersion, newVersion, err) + } + rowsAffected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("failed to get queue summary update rows request_id=%s: %w", summary.RequestID, err) + } + if rowsAffected != 1 { + return fmt.Errorf("queue summary request_id=%s expected_version=%d: %w", summary.RequestID, oldVersion, storage.ErrVersionMismatch) + } + return nil +} + +func (s *requestQueueSummaryStore) List(ctx context.Context, query storage.RequestQueueSummaryQuery) (ret []entity.RequestQueueSummary, retErr error) { + op := metrics.Begin(s.scope, "list") + defer func() { op.Complete(retErr) }() + + statement := ` + SELECT queue, received_at_ms, request_id, change_uris, status, + version, last_error, metadata + FROM request_summary_by_queue + WHERE queue = ? AND received_at_ms >= ? AND received_at_ms < ?` + args := []any{query.Queue, query.ReceivedAtOrAfterMs, query.ReceivedBeforeMs} + if query.HasCursor { + statement += " AND (received_at_ms < ? OR (received_at_ms = ? AND request_id < ?))" + args = append(args, query.Cursor.ReceivedAtMs, query.Cursor.ReceivedAtMs, query.Cursor.RequestID) + } + statement += " ORDER BY received_at_ms DESC, request_id DESC LIMIT ?" + args = append(args, query.Limit) + + rows, err := s.db.QueryContext(ctx, statement, args...) + if err != nil { + return nil, fmt.Errorf("failed to list queue summaries queue=%s: %w", query.Queue, err) + } + defer rows.Close() + + results := make([]entity.RequestQueueSummary, 0) + for rows.Next() { + var summary entity.RequestQueueSummary + var changeURIsJSON []byte + var metadataJSON []byte + if err := rows.Scan(&summary.Queue, &summary.ReceivedAtMs, &summary.RequestID, &changeURIsJSON, &summary.Status, &summary.Version, &summary.LastError, &metadataJSON); err != nil { + return nil, fmt.Errorf("failed to scan queue summary queue=%s: %w", query.Queue, err) + } + if err := unmarshalSummaryJSON(changeURIsJSON, metadataJSON, &summary.ChangeURIs, &summary.Metadata); err != nil { + return nil, fmt.Errorf("failed to decode queue summary request_id=%s: %w", summary.RequestID, err) + } + results = append(results, summary) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to iterate queue summaries queue=%s: %w", query.Queue, err) + } + return results, nil +} diff --git a/submitqueue/extension/storage/mysql/request_store.go b/submitqueue/extension/storage/mysql/request_store.go index d332498d..fd542c9d 100644 --- a/submitqueue/extension/storage/mysql/request_store.go +++ b/submitqueue/extension/storage/mysql/request_store.go @@ -84,9 +84,7 @@ func (r *requestStore) Create(ctx context.Context, request entity.Request) (retE ) if err != nil { var mysqlErr *mysql.MySQLError - if errors.As(err, &mysqlErr) && mysqlErr.Number == 1062 { - // MySQL error code 1062 is "Duplicate entry". Hopefully it will never change with new versions of MySQL. - // Also it requires to have a single unique index on the table. + if errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlErrDuplicateEntry { return fmt.Errorf("request entity id=%s: %w", request.ID, storage.ErrAlreadyExists) } return fmt.Errorf("failed to insert request entity id=%s: %w", request.ID, err) diff --git a/submitqueue/extension/storage/mysql/request_summary_store.go b/submitqueue/extension/storage/mysql/request_summary_store.go new file mode 100644 index 00000000..36849dea --- /dev/null +++ b/submitqueue/extension/storage/mysql/request_summary_store.go @@ -0,0 +1,171 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysql + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + + "github.com/go-sql-driver/mysql" + "github.com/uber-go/tally" + + "github.com/uber/submitqueue/platform/metrics" + "github.com/uber/submitqueue/submitqueue/entity" + "github.com/uber/submitqueue/submitqueue/extension/storage" +) + +type requestSummaryStore struct { + db *sql.DB + scope tally.Scope +} + +// NewRequestSummaryStore creates a MySQL-backed RequestSummaryStore. +func NewRequestSummaryStore(db *sql.DB, scope tally.Scope) storage.RequestSummaryStore { + return &requestSummaryStore{db: db, scope: scope} +} + +func (s *requestSummaryStore) Create(ctx context.Context, summary entity.RequestSummary) (retErr error) { + op := metrics.Begin(s.scope, "create") + defer func() { op.Complete(retErr) }() + + changeURIsJSON, metadataJSON, err := marshalSummaryJSON(summary.ChangeURIs, summary.Metadata) + if err != nil { + return fmt.Errorf("failed to marshal request summary request_id=%s: %w", summary.RequestID, err) + } + + _, err = s.db.ExecContext(ctx, ` + INSERT INTO request_summary ( + request_id, queue, change_uris, received_at_ms, status, request_version, + status_timestamp_ms, version, + last_error, metadata + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + summary.RequestID, summary.Queue, changeURIsJSON, summary.ReceivedAtMs, summary.Status, + summary.RequestVersion, summary.StatusTimestampMs, summary.Version, + summary.LastError, metadataJSON, + ) + if err != nil { + var mysqlErr *mysql.MySQLError + if errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlErrDuplicateEntry { + return fmt.Errorf("request summary request_id=%s: %w", summary.RequestID, storage.ErrAlreadyExists) + } + return fmt.Errorf("failed to insert request summary request_id=%s: %w", summary.RequestID, err) + } + + return nil +} + +func (s *requestSummaryStore) Get(ctx context.Context, requestID string) (ret entity.RequestSummary, retErr error) { + op := metrics.Begin(s.scope, "get") + defer func() { op.Complete(retErr) }() + + var changeURIsJSON []byte + var metadataJSON []byte + err := s.db.QueryRowContext(ctx, ` + SELECT request_id, queue, change_uris, received_at_ms, status, request_version, + status_timestamp_ms, version, + last_error, metadata + FROM request_summary + WHERE request_id = ?`, requestID, + ).Scan( + &ret.RequestID, &ret.Queue, &changeURIsJSON, &ret.ReceivedAtMs, &ret.Status, + &ret.RequestVersion, &ret.StatusTimestampMs, &ret.Version, + &ret.LastError, &metadataJSON, + ) + if errors.Is(err, sql.ErrNoRows) { + return entity.RequestSummary{}, storage.WrapNotFound(err) + } + if err != nil { + return entity.RequestSummary{}, fmt.Errorf("failed to get request summary request_id=%s: %w", requestID, err) + } + if err := unmarshalSummaryJSON(changeURIsJSON, metadataJSON, &ret.ChangeURIs, &ret.Metadata); err != nil { + return entity.RequestSummary{}, fmt.Errorf("failed to decode request summary request_id=%s: %w", requestID, err) + } + + return ret, nil +} + +func (s *requestSummaryStore) Update(ctx context.Context, summary entity.RequestSummary, oldVersion, newVersion int32) (retErr error) { + op := metrics.Begin(s.scope, "update") + defer func() { op.Complete(retErr) }() + + metadata := normalizeMetadata(summary.Metadata) + metadataJSON, err := json.Marshal(metadata) + if err != nil { + return fmt.Errorf("failed to marshal request summary metadata request_id=%s: %w", summary.RequestID, err) + } + + result, err := s.db.ExecContext(ctx, ` + UPDATE request_summary + SET status = ?, request_version = ?, status_timestamp_ms = ?, + version = ?, last_error = ?, metadata = ? + WHERE request_id = ? AND version = ?`, + summary.Status, summary.RequestVersion, summary.StatusTimestampMs, + newVersion, summary.LastError, metadataJSON, + summary.RequestID, oldVersion, + ) + if err != nil { + return fmt.Errorf("failed to update request summary request_id=%s old_version=%d new_version=%d: %w", summary.RequestID, oldVersion, newVersion, err) + } + rowsAffected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("failed to get request summary update rows request_id=%s: %w", summary.RequestID, err) + } + if rowsAffected != 1 { + return fmt.Errorf("request summary request_id=%s expected_version=%d: %w", summary.RequestID, oldVersion, storage.ErrVersionMismatch) + } + + return nil +} + +func marshalSummaryJSON(changeURIs []string, metadata map[string]string) ([]byte, []byte, error) { + changeURIsJSON, err := json.Marshal(normalizeChangeURIs(changeURIs)) + if err != nil { + return nil, nil, err + } + metadataJSON, err := json.Marshal(normalizeMetadata(metadata)) + if err != nil { + return nil, nil, err + } + return changeURIsJSON, metadataJSON, nil +} + +func unmarshalSummaryJSON(changeURIsJSON, metadataJSON []byte, changeURIs *[]string, metadata *map[string]string) error { + if err := json.Unmarshal(changeURIsJSON, changeURIs); err != nil { + return err + } + if err := json.Unmarshal(metadataJSON, metadata); err != nil { + return err + } + *changeURIs = normalizeChangeURIs(*changeURIs) + *metadata = normalizeMetadata(*metadata) + return nil +} + +func normalizeChangeURIs(changeURIs []string) []string { + if changeURIs == nil { + return []string{} + } + return changeURIs +} + +func normalizeMetadata(metadata map[string]string) map[string]string { + if metadata == nil { + return map[string]string{} + } + return metadata +} diff --git a/submitqueue/extension/storage/mysql/request_uri_store.go b/submitqueue/extension/storage/mysql/request_uri_store.go new file mode 100644 index 00000000..35375d5c --- /dev/null +++ b/submitqueue/extension/storage/mysql/request_uri_store.go @@ -0,0 +1,86 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysql + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/go-sql-driver/mysql" + "github.com/uber-go/tally" + + "github.com/uber/submitqueue/platform/metrics" + "github.com/uber/submitqueue/submitqueue/entity" + "github.com/uber/submitqueue/submitqueue/extension/storage" +) + +type requestURIStore struct { + db *sql.DB + scope tally.Scope +} + +// NewRequestURIStore creates a MySQL-backed RequestURIStore. +func NewRequestURIStore(db *sql.DB, scope tally.Scope) storage.RequestURIStore { + return &requestURIStore{db: db, scope: scope} +} + +func (s *requestURIStore) Create(ctx context.Context, mapping entity.RequestURI) (retErr error) { + op := metrics.Begin(s.scope, "create") + defer func() { op.Complete(retErr) }() + + _, err := s.db.ExecContext(ctx, + "INSERT INTO change_uri_request_mapping (change_uri, received_at_ms, request_id) VALUES (?, ?, ?)", + mapping.ChangeURI, mapping.ReceivedAtMs, mapping.RequestID, + ) + if err != nil { + var mysqlErr *mysql.MySQLError + if errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlErrDuplicateEntry { + return fmt.Errorf("request URI change_uri=%s received_at_ms=%d request_id=%s: %w", mapping.ChangeURI, mapping.ReceivedAtMs, mapping.RequestID, storage.ErrAlreadyExists) + } + return fmt.Errorf("failed to insert request URI request_id=%s change_uri=%s: %w", mapping.RequestID, mapping.ChangeURI, err) + } + return nil +} + +func (s *requestURIStore) ListByURI(ctx context.Context, changeURI string, limit int) (ret []entity.RequestURI, retErr error) { + op := metrics.Begin(s.scope, "list_by_uri") + defer func() { op.Complete(retErr) }() + + rows, err := s.db.QueryContext(ctx, ` + SELECT change_uri, received_at_ms, request_id + FROM change_uri_request_mapping + WHERE change_uri = ? + ORDER BY received_at_ms DESC, request_id DESC + LIMIT ?`, changeURI, limit) + if err != nil { + return nil, fmt.Errorf("failed to list request URIs change_uri=%s: %w", changeURI, err) + } + defer rows.Close() + + results := make([]entity.RequestURI, 0) + for rows.Next() { + var mapping entity.RequestURI + if err := rows.Scan(&mapping.ChangeURI, &mapping.ReceivedAtMs, &mapping.RequestID); err != nil { + return nil, fmt.Errorf("failed to scan request URI change_uri=%s: %w", changeURI, err) + } + results = append(results, mapping) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to iterate request URIs change_uri=%s: %w", changeURI, err) + } + return results, nil +} diff --git a/submitqueue/extension/storage/mysql/speculation_tree_store.go b/submitqueue/extension/storage/mysql/speculation_tree_store.go index 7592ce94..3245eef5 100644 --- a/submitqueue/extension/storage/mysql/speculation_tree_store.go +++ b/submitqueue/extension/storage/mysql/speculation_tree_store.go @@ -82,7 +82,7 @@ func (s *speculationTreeStore) Create(ctx context.Context, speculationTree entit ) if err != nil { var mysqlErr *mysql.MySQLError - if errors.As(err, &mysqlErr) && mysqlErr.Number == 1062 { + if errors.As(err, &mysqlErr) && mysqlErr.Number == mysqlErrDuplicateEntry { return fmt.Errorf("speculation tree entity batchID=%s: %w", speculationTree.BatchID, storage.ErrAlreadyExists) } return fmt.Errorf("failed to insert speculation tree entity batchID=%s: %w", speculationTree.BatchID, err) diff --git a/submitqueue/extension/storage/mysql/storage.go b/submitqueue/extension/storage/mysql/storage.go index 40e128f2..ffcf4d02 100644 --- a/submitqueue/extension/storage/mysql/storage.go +++ b/submitqueue/extension/storage/mysql/storage.go @@ -23,6 +23,10 @@ import ( "github.com/uber/submitqueue/submitqueue/extension/storage" ) +// mysqlErrDuplicateEntry is MySQL error code 1062 ("Duplicate entry"), returned on a unique or primary key violation. +// It requires a unique index on the table to be raised. +const mysqlErrDuplicateEntry = 1062 + type mysqlStorage struct { db *sql.DB requestStore storage.RequestStore @@ -33,6 +37,9 @@ type mysqlStorage struct { speculationPathBuildStore storage.SpeculationPathBuildStore speculationTreeStore storage.SpeculationTreeStore requestLogStore storage.RequestLogStore + requestSummaryStore storage.RequestSummaryStore + requestQueueStore storage.RequestQueueSummaryStore + requestURIStore storage.RequestURIStore } // NewStorage creates a new MySQL storage. @@ -47,6 +54,9 @@ func NewStorage(db *sql.DB, scope tally.Scope) (storage.Storage, error) { speculationPathBuildStore: NewSpeculationPathBuildStore(db, scope.SubScope("speculation_path_build_store")), speculationTreeStore: NewSpeculationTreeStore(db, scope.SubScope("speculation_tree_store")), requestLogStore: NewRequestLogStore(db, scope.SubScope("request_log_store")), + requestSummaryStore: NewRequestSummaryStore(db, scope.SubScope("request_summary_store")), + requestQueueStore: NewRequestQueueSummaryStore(db, scope.SubScope("request_queue_summary_store")), + requestURIStore: NewRequestURIStore(db, scope.SubScope("request_uri_store")), }, nil } @@ -90,6 +100,21 @@ func (f *mysqlStorage) GetRequestLogStore() storage.RequestLogStore { return f.requestLogStore } +// GetRequestSummaryStore returns the MySQL-backed RequestSummaryStore. +func (f *mysqlStorage) GetRequestSummaryStore() storage.RequestSummaryStore { + return f.requestSummaryStore +} + +// GetRequestQueueSummaryStore returns the MySQL-backed RequestQueueSummaryStore. +func (f *mysqlStorage) GetRequestQueueSummaryStore() storage.RequestQueueSummaryStore { + return f.requestQueueStore +} + +// GetRequestURIStore returns the MySQL-backed RequestURIStore. +func (f *mysqlStorage) GetRequestURIStore() storage.RequestURIStore { + return f.requestURIStore +} + // Close closes the underlying database connection. func (f *mysqlStorage) Close() error { return f.db.Close() diff --git a/submitqueue/extension/storage/request_queue_summary_store.go b/submitqueue/extension/storage/request_queue_summary_store.go new file mode 100644 index 00000000..68acf211 --- /dev/null +++ b/submitqueue/extension/storage/request_queue_summary_store.go @@ -0,0 +1,63 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +//go:generate mockgen -source=request_queue_summary_store.go -destination=mock/request_queue_summary_store_mock.go -package=mock + +import ( + "context" + + "github.com/uber/submitqueue/submitqueue/entity" +) + +// RequestQueueSummaryCursor is the exclusive keyset boundary for a descending queue-summary query. +type RequestQueueSummaryCursor struct { + // ReceivedAtMs is the receipt timestamp of the last item from the previous page. + ReceivedAtMs int64 + // RequestID is the request ID of the last item from the previous page. + RequestID string +} + +// RequestQueueSummaryQuery specifies one bounded queue-summary page query. +type RequestQueueSummaryQuery struct { + // Queue is the exact queue partition to scan. + Queue string + // ReceivedAtOrAfterMs is the inclusive lower receipt-time bound. + ReceivedAtOrAfterMs int64 + // ReceivedBeforeMs is the exclusive upper receipt-time bound. + ReceivedBeforeMs int64 + // Cursor is an exclusive continuation boundary when HasCursor is true. + Cursor RequestQueueSummaryCursor + // HasCursor selects whether Cursor participates in the query. + HasCursor bool + // Limit is the maximum number of rows returned and must be positive. + Limit int +} + +// RequestQueueSummaryStore persists the queue-ordered request projection. +type RequestQueueSummaryStore interface { + // Create inserts summary and returns ErrAlreadyExists when its full primary key already exists. + Create(ctx context.Context, summary entity.RequestQueueSummary) error + + // Get returns the row identified by its full primary key, or ErrNotFound when absent. + Get(ctx context.Context, queue string, receivedAtMs int64, requestID string) (entity.RequestQueueSummary, error) + + // Update conditionally replaces mutable fields when the persisted projection version equals oldVersion. + // The store writes newVersion exactly as supplied and returns ErrVersionMismatch when the guard does not match. + Update(ctx context.Context, summary entity.RequestQueueSummary, oldVersion, newVersion int32) error + + // List returns at most query.Limit rows ordered by received_at_ms descending, then request_id descending. + List(ctx context.Context, query RequestQueueSummaryQuery) ([]entity.RequestQueueSummary, error) +} diff --git a/submitqueue/extension/storage/request_summary_store.go b/submitqueue/extension/storage/request_summary_store.go new file mode 100644 index 00000000..5e2a657a --- /dev/null +++ b/submitqueue/extension/storage/request_summary_store.go @@ -0,0 +1,37 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +//go:generate mockgen -source=request_summary_store.go -destination=mock/request_summary_store_mock.go -package=mock + +import ( + "context" + + "github.com/uber/submitqueue/submitqueue/entity" +) + +// RequestSummaryStore persists the authoritative request-ID materialized view. +type RequestSummaryStore interface { + // Create inserts summary and returns ErrAlreadyExists when RequestID already exists. + // The caller owns retry identity and decides whether an existing row is an identical retry or a conflict. + Create(ctx context.Context, summary entity.RequestSummary) error + + // Get returns the summary for requestID, or ErrNotFound when absent. + Get(ctx context.Context, requestID string) (entity.RequestSummary, error) + + // Update conditionally replaces the mutable status fields when the persisted projection version equals oldVersion. + // The store writes newVersion exactly as supplied and returns ErrVersionMismatch when the guard does not match. + Update(ctx context.Context, summary entity.RequestSummary, oldVersion, newVersion int32) error +} diff --git a/submitqueue/extension/storage/request_uri_store.go b/submitqueue/extension/storage/request_uri_store.go new file mode 100644 index 00000000..9ac1c024 --- /dev/null +++ b/submitqueue/extension/storage/request_uri_store.go @@ -0,0 +1,32 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +//go:generate mockgen -source=request_uri_store.go -destination=mock/request_uri_store_mock.go -package=mock + +import ( + "context" + + "github.com/uber/submitqueue/submitqueue/entity" +) + +// RequestURIStore persists the immutable change-URI reverse mapping. +type RequestURIStore interface { + // Create inserts mapping and returns ErrAlreadyExists when its full primary key already exists. + Create(ctx context.Context, mapping entity.RequestURI) error + + // ListByURI returns at most limit mappings ordered by received_at_ms descending, then request_id descending. + ListByURI(ctx context.Context, changeURI string, limit int) ([]entity.RequestURI, error) +} diff --git a/submitqueue/extension/storage/storage.go b/submitqueue/extension/storage/storage.go index a1ec91ef..b54736c3 100644 --- a/submitqueue/extension/storage/storage.go +++ b/submitqueue/extension/storage/storage.go @@ -68,6 +68,15 @@ type Storage interface { // GetRequestLogStore returns the RequestLogStore instance. GetRequestLogStore() RequestLogStore + // GetRequestSummaryStore returns the RequestSummaryStore instance. + GetRequestSummaryStore() RequestSummaryStore + + // GetRequestQueueSummaryStore returns the RequestQueueSummaryStore instance. + GetRequestQueueSummaryStore() RequestQueueSummaryStore + + // GetRequestURIStore returns the RequestURIStore instance. + GetRequestURIStore() RequestURIStore + // Close closes the storage and all underlying connections. Should only be called once at the end of the program. Close() error } diff --git a/test/integration/submitqueue/extension/storage/suite.go b/test/integration/submitqueue/extension/storage/suite.go index 866e4f94..a07d7cbe 100644 --- a/test/integration/submitqueue/extension/storage/suite.go +++ b/test/integration/submitqueue/extension/storage/suite.go @@ -588,3 +588,124 @@ func (s *StorageContractSuite) TestStorage_SpeculationPathBuildGetNotFound() { _, err := s.storage.GetSpeculationPathBuildStore().Get(ctx, "spec/path-build/nonexistent") assert.ErrorIs(t, err, storage.ErrNotFound, "Get for unknown path ID should return ErrNotFound") } + +func (s *StorageContractSuite) TestStorage_RequestSummaryCreateGetAndCAS() { + t := s.T() + ctx := s.ctx + summary := entity.RequestSummary{ + RequestID: "summary/1", Queue: "summary-q", ChangeURIs: nil, ReceivedAtMs: 100, + Status: entity.RequestStatusAccepted, StatusTimestampMs: 100, Version: 1, Metadata: nil, + } + + require.NoError(t, s.storage.GetRequestSummaryStore().Create(ctx, summary)) + require.ErrorIs(t, s.storage.GetRequestSummaryStore().Create(ctx, summary), storage.ErrAlreadyExists) + + got, err := s.storage.GetRequestSummaryStore().Get(ctx, summary.RequestID) + require.NoError(t, err) + assert.NotNil(t, got.ChangeURIs) + assert.NotNil(t, got.Metadata) + _, err = s.storage.GetRequestSummaryStore().Get(ctx, "summary/missing") + require.ErrorIs(t, err, storage.ErrNotFound) + + got.Status = entity.RequestStatusLanded + got.RequestVersion = 2 + got.StatusTimestampMs = 200 + got.LastError = "terminal detail" + got.Metadata = map[string]string{"source": "test"} + require.NoError(t, s.storage.GetRequestSummaryStore().Update(ctx, got, 1, 2)) + require.ErrorIs(t, s.storage.GetRequestSummaryStore().Update(ctx, got, 1, 3), storage.ErrVersionMismatch) + + updated, err := s.storage.GetRequestSummaryStore().Get(ctx, summary.RequestID) + require.NoError(t, err) + assert.Equal(t, int32(2), updated.Version) + assert.Equal(t, entity.RequestStatusLanded, updated.Status) + assert.Equal(t, "terminal detail", updated.LastError) + assert.Equal(t, map[string]string{"source": "test"}, updated.Metadata) +} + +func (s *StorageContractSuite) TestStorage_RequestQueueSummaryListAndCursor() { + t := s.T() + ctx := s.ctx + store := s.storage.GetRequestQueueSummaryStore() + rows := []entity.RequestQueueSummary{ + {RequestID: "queue-summary/1", Queue: "queue-summary", ChangeURIs: nil, ReceivedAtMs: 100, Status: entity.RequestStatusAccepted, Version: 1, Metadata: nil}, + {RequestID: "queue-summary/2", Queue: "queue-summary", ChangeURIs: []string{"uri/2"}, ReceivedAtMs: 200, Status: entity.RequestStatusLanded, Version: 1, Metadata: map[string]string{}}, + {RequestID: "queue-summary/3", Queue: "queue-summary", ChangeURIs: []string{"uri/3"}, ReceivedAtMs: 200, Status: entity.RequestStatusError, Version: 1, Metadata: map[string]string{}}, + } + for _, row := range rows { + require.NoError(t, store.Create(ctx, row)) + } + require.ErrorIs(t, store.Create(ctx, rows[0]), storage.ErrAlreadyExists) + + got, err := store.Get(ctx, rows[0].Queue, rows[0].ReceivedAtMs, rows[0].RequestID) + require.NoError(t, err) + assert.NotNil(t, got.ChangeURIs) + assert.NotNil(t, got.Metadata) + _, err = store.Get(ctx, "queue-summary", 999, "queue-summary/missing") + require.ErrorIs(t, err, storage.ErrNotFound) + + got.Status = entity.RequestStatusLanded + got.LastError = "done" + got.Metadata = map[string]string{"result": "landed"} + require.NoError(t, store.Update(ctx, got, 1, 2)) + require.ErrorIs(t, store.Update(ctx, got, 1, 3), storage.ErrVersionMismatch) + updated, err := store.Get(ctx, got.Queue, got.ReceivedAtMs, got.RequestID) + require.NoError(t, err) + assert.Equal(t, int32(2), updated.Version) + assert.Equal(t, entity.RequestStatusLanded, updated.Status) + assert.Equal(t, "done", updated.LastError) + + firstPage, err := store.List(ctx, storage.RequestQueueSummaryQuery{ + Queue: "queue-summary", ReceivedAtOrAfterMs: 50, ReceivedBeforeMs: 250, Limit: 2, + }) + require.NoError(t, err) + require.Len(t, firstPage, 2) + assert.Equal(t, []string{"queue-summary/3", "queue-summary/2"}, []string{firstPage[0].RequestID, firstPage[1].RequestID}) + + secondPage, err := store.List(ctx, storage.RequestQueueSummaryQuery{ + Queue: "queue-summary", ReceivedAtOrAfterMs: 50, ReceivedBeforeMs: 250, Limit: 2, + HasCursor: true, Cursor: storage.RequestQueueSummaryCursor{ReceivedAtMs: 200, RequestID: "queue-summary/2"}, + }) + require.NoError(t, err) + require.Len(t, secondPage, 1) + assert.Equal(t, "queue-summary/1", secondPage[0].RequestID) + assert.NotNil(t, secondPage[0].ChangeURIs) + assert.NotNil(t, secondPage[0].Metadata) + + bounded, err := store.List(ctx, storage.RequestQueueSummaryQuery{ + Queue: "queue-summary", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, Limit: 10, + }) + require.NoError(t, err) + require.Len(t, bounded, 1) + assert.Equal(t, "queue-summary/1", bounded[0].RequestID) + + empty, err := store.List(ctx, storage.RequestQueueSummaryQuery{ + Queue: "queue-summary", ReceivedAtOrAfterMs: 300, ReceivedBeforeMs: 400, Limit: 10, + }) + require.NoError(t, err) + assert.Empty(t, empty) +} + +func (s *StorageContractSuite) TestStorage_RequestURIListIsBoundedAndOrdered() { + t := s.T() + ctx := s.ctx + store := s.storage.GetRequestURIStore() + rows := []entity.RequestURI{ + {ChangeURI: "uri/shared", ReceivedAtMs: 100, RequestID: "uri/1"}, + {ChangeURI: "uri/shared", ReceivedAtMs: 200, RequestID: "uri/2"}, + {ChangeURI: "uri/shared", ReceivedAtMs: 200, RequestID: "uri/3"}, + } + for _, row := range rows { + require.NoError(t, store.Create(ctx, row)) + } + require.ErrorIs(t, store.Create(ctx, rows[0]), storage.ErrAlreadyExists) + + got, err := store.ListByURI(ctx, "uri/shared", 2) + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, []string{"uri/3", "uri/2"}, []string{got[0].RequestID, got[1].RequestID}) + + empty, err := store.ListByURI(ctx, "uri/missing", 2) + require.NoError(t, err) + assert.Empty(t, empty) +}