github::ActivityMonitor method backfill_locked (protected)
<instance of github::ActivityMonitor> backfill_locked \ [ -repo repo ] [ -branch branch ] [ -revision revision ] \ [ -base base ] [ -since since ] [ -start_page start_page ] \ [ -max_pages max_pages ]
Defined in packages/xowiki/tcl/github-activity-monitor-procs.tcl
Backfill commit history for a single repo/branch using /repos/{repo}/commits. Uses synthetic negative event_id values. repo e.g. "openacs/openacs-core" branch e.g. "main" or "oacs-5-10"
- Switches:
- -repo (optional)
- -branch (optional, defaults to
"main")- -revision (optional)
- -base (optional)
- -since (optional)
- -start_page (optional, defaults to
"1")- -max_pages (optional, defaults to
"50")- Testcases:
- No testcase defined.
Source code: if {![string is integer -strict $start_page] || $start_page < 1 || ![string is integer -strict $max_pages] || $max_pages < 1} { error "start_page and max_pages must be positive integers" } if {$revision eq ""} { set revision $branch } ns_log Notice "GitHub backfill_repo_history: $repo ($branch)" # We’ll assign event_id from next negative downwards. set next_event_id [:next_backfill_event_id] set page $start_page set pages_done 0 set done 0 set count 0 while {!$done} { if {$pages_done >= $max_pages} { ns_log notice "GitHub backfill_repo_history: reached" "max_pages=$max_pages for $repo ($branch)" break } # Compare against the successfully reconciled head. Do not # simply break when that SHA appears in a date-ordered commit # list: newly merged side branches may have older timestamps. set batch "" set full_history [expr {$base eq ""}] if {!$full_history} { if {[catch { set comparison [::util::json2dict [:get "/repos/$repo/compare/${base}...${revision}" [list page $page per_page 20]]] } message options]} { if {[dict get $options -errorcode] ne {GITHUB HTTP 404}} { return -options $options $message } # An unavailable comparison must not silently advance # the checkpoint or trigger an unbounded history scan. set attention [:get_state attention_required {}] dict set attention [list $repo $branch] [dict create base $base revision $revision reason missing-comparison-base] :set_state attention_required $attention return -code error -errorcode {GITHUB BASE_UNAVAILABLE} "Cannot compare $repo $branch with saved base $base; job retained, no full reload attempted" } elseif {[dict get $comparison status] in {ahead identical diverged behind}} { set batch [dict get $comparison commits] } else { error "Unexpected comparison status: [dict get $comparison status]" } } if {$full_history} { set query [list sha $revision page $page per_page 20] if {$since ne ""} {lappend query since $since} set body [:get "/repos/$repo/commits" $query] set batch [::util::json2dict $body] } if {[llength $batch] == 0} { ns_log notice "GitHub backfill_repo_history: no more commits" at page $page for $repo ($branch) set done 1 break } # The public entry point owns the transaction and lock. # Avoid nested transaction abort semantics on budget exhaustion. foreach summary $batch { set sha [dict get $summary sha] # Skip duplicates; a commit shared with another branch # does not prove that all older history is present. set exists [xo::dc 0or1row exists_commit { select 1 from github_activity where repo = :repo and sha = :sha limit 1 }] if {$exists} { continue } incr count # # Fetch full commit details (stats, files, etc.), # similar to summarize_push_event. # set commit_body [:get "/repos/$repo/commits/$sha"] set c [::util::json2dict $commit_body] set commit [dict get $c commit] #ns_log notice "commit keys: [dict keys $commit]" ns_log notice "(page $page, count $count) commit.message($count):" [dict get $commit message] set author_dict [dict get $commit author] set author_name [dict get $author_dict name] set commit_date [dict get $author_dict date] set stats [dict get $c stats] set additions [dict get $stats additions] set deletions [dict get $stats deletions] set files [dict get $c files] set files_changed [llength $files] set message_full [dict get $commit message] if {$message_full eq ""} { # Fallback: don’t leave title NULL, DB wants NOT NULL set title "(no commit message)" } else { set title [lindex [split [string trimleft $message_full] \n] 0] } #ns_log notice "commit.title: $title" if {[dict exists $c html_url]} { set url [dict get $c html_url] } else { # construct GitHub web URL as a fallback set url "https://github.com/$repo/commit/$sha" } # Synthetic negative event_id set event_id $next_event_id incr next_event_id -1 set created_at $commit_date ;# for backfill, we just reuse commit time ::xo::dc dml insert_backfill { insert into github_activity ( event_id, repo, branch, sha, author_name, author_login, commit_date, created_at, title, url, files_changed, additions, deletions, raw_payload ) values ( :event_id, :repo, :branch, :sha, :author_name, NULL, :commit_date, :created_at, :title, :url, :files_changed, :additions, :deletions, :commit_body ) on conflict (repo, sha) do nothing } } if {$done} { break } incr page incr pages_done if {[llength $batch] < 20} { set done 1 } } ns_log notice "GitHub backfill_repo_history: done for $repo ($branch)" return [dict create complete $done next_page $page base $base]XQL Not present: Generic, PostgreSQL, Oracle
![[i]](/resources/acs-subsite/ZoomIn16.gif)