github::ActivityMonitor method backfill_locked (protected)

 <instance of github::ActivityMonitor[i]> backfill_locked \
    [ -repo repo ] [ -branch branch ] [ -revision revision ] \
    [ -base base ] [ -since since ] [ -start_page start_page ] \
    [ -max_pages max_pages ]

Defined in packages/xowiki/tcl/github-activity-monitor-procs.tcl

Backfill commit history for a single repo/branch using /repos/{repo}/commits. Uses synthetic negative event_id values. repo e.g. "openacs/openacs-core" branch e.g. "main" or "oacs-5-10"

Switches:
-repo (optional)
-branch (optional, defaults to "main")
-revision (optional)
-base (optional)
-since (optional)
-start_page (optional, defaults to "1")
-max_pages (optional, defaults to "50")

Testcases:
No testcase defined.
Source code:
if {![string is integer -strict $start_page] || $start_page < 1
    || ![string is integer -strict $max_pages] || $max_pages < 1} {
    error "start_page and max_pages must be positive integers"
}
if {$revision eq ""} {
    set revision $branch
}
ns_log Notice "GitHub backfill_repo_history: $repo ($branch)"

# We’ll assign event_id from next negative downwards.
set next_event_id [:next_backfill_event_id]

set page $start_page
set pages_done 0
set done 0
set count 0

while {!$done} {
    if {$pages_done >= $max_pages} {
        ns_log notice "GitHub backfill_repo_history: reached"  "max_pages=$max_pages for $repo ($branch)"
        break
    }

    # Compare against the successfully reconciled head. Do not
    # simply break when that SHA appears in a date-ordered commit
    # list: newly merged side branches may have older timestamps.
    set batch ""
    set full_history [expr {$base eq ""}]
    if {!$full_history} {
        if {[catch {
            set comparison [::util::json2dict [:get "/repos/$repo/compare/${base}...${revision}"  [list page $page per_page 20]]]
        } message options]} {
            if {[dict get $options -errorcode] ne {GITHUB HTTP 404}} {
                return -options $options $message
            }
            # An unavailable comparison must not silently advance
            # the checkpoint or trigger an unbounded history scan.
            set attention [:get_state attention_required {}]
            dict set attention [list $repo $branch] [dict create base $base revision $revision reason missing-comparison-base]
            :set_state attention_required $attention
            return -code error -errorcode {GITHUB BASE_UNAVAILABLE} "Cannot compare $repo $branch with saved base $base; job retained, no full reload attempted"
        } elseif {[dict get $comparison status] in {ahead identical diverged behind}} {
            set batch [dict get $comparison commits]
        } else {
            error "Unexpected comparison status: [dict get $comparison status]"
        }
    }
    if {$full_history} {
        set query [list sha $revision page $page per_page 20]
        if {$since ne ""} {lappend query since $since}
        set body [:get "/repos/$repo/commits" $query]
        set batch [::util::json2dict $body]
    }

    if {[llength $batch] == 0} {
        ns_log notice "GitHub backfill_repo_history: no more commits"  at page $page for $repo ($branch)
        set done 1
        break
    }

    # The public entry point owns the transaction and lock.
    # Avoid nested transaction abort semantics on budget exhaustion.
    foreach summary $batch {
        set sha [dict get $summary sha]

        # Skip duplicates; a commit shared with another branch
        # does not prove that all older history is present.
        set exists [xo::dc 0or1row exists_commit {
            select 1 from github_activity where repo = :repo and sha = :sha limit 1
        }]

        if {$exists} {
            continue
        }

        incr count

        #
        # Fetch full commit details (stats, files, etc.),
        # similar to summarize_push_event.
        #
        set commit_body   [:get "/repos/$repo/commits/$sha"]
        set c             [::util::json2dict $commit_body]
        set commit        [dict get $c commit]

        #ns_log notice "commit keys: [dict keys $commit]"
        ns_log notice "(page $page, count $count) commit.message($count):"  [dict get $commit message]

        set author_dict   [dict get $commit author]
        set author_name   [dict get $author_dict name]
        set commit_date   [dict get $author_dict date]
        set stats         [dict get $c stats]
        set additions     [dict get $stats additions]
        set deletions     [dict get $stats deletions]
        set files         [dict get $c files]
        set files_changed [llength $files]

        set message_full  [dict get $commit message]

        if {$message_full eq ""} {
            # Fallback: don’t leave title NULL, DB wants NOT NULL
            set title "(no commit message)"
        } else {
            set title [lindex [split [string trimleft $message_full] \n] 0]
        }
        #ns_log notice "commit.title: $title"

        if {[dict exists $c html_url]} {
            set url [dict get $c html_url]
        } else {
            # construct GitHub web URL as a fallback
            set url "https://github.com/$repo/commit/$sha"
        }

        # Synthetic negative event_id
        set event_id $next_event_id
        incr next_event_id -1

        set created_at $commit_date   ;# for backfill, we just reuse commit time

        ::xo::dc dml insert_backfill {
            insert into github_activity (
                                         event_id, repo, branch, sha,
                                         author_name, author_login,
                                         commit_date, created_at,
                                         title, url,
                                         files_changed, additions, deletions,
                                         raw_payload
                                         ) values (
                                                   :event_id, :repo, :branch, :sha,
                                                   :author_name, NULL,
                                                   :commit_date, :created_at,
                                                   :title, :url,
                                                   :files_changed, :additions, :deletions,
                                                   :commit_body
                                                   )
            on conflict (repo, sha) do nothing
        }
    }

    if {$done} {
        break
    }

    incr page
    incr pages_done
    if {[llength $batch] < 20} {
        set done 1
    }
}

ns_log notice "GitHub backfill_repo_history: done for $repo ($branch)"
return [dict create complete $done next_page $page base $base]
XQL Not present:
Generic, PostgreSQL, Oracle
[ hide source ] | [ make this the default ]
Show another procedure: