ham-ref/cloudlog
1
0
Fork 0
mirror of https://github.com/magicbug/Cloudlog synced 2026-08-13 17:49:35 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add LoTW batch updates and performance indexes

Browse source 
Bump migration version to 247 and add a migration that creates performance indexes for LoTW workflows (idx_lotw_qslrdate, idx_lotw_qsos_to_upload, idx_lotw_confirmation_match) to speed common queries. Refactor Lotw controller to collect records in memory, build rows for rendering, and perform a single batch confirmation update via a new Logbook_model::lotw_update_batch() method instead of many individual updates. Implement lotw_update_batch() to look up matching QSOs, perform update_batch() calls, and separately update gridsquares/distances using the Qra library. Simplify LotwCert::toggle_archive_certificate() to toggle archived state with a single UPDATE and return the resulting status.
This commit is contained in:
Peter Goodhall 2026-02-11 11:14:59 +00:00
parent 2cb8d62860
commit cfa24d9f1a
5 changed files with 350 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

2
application/config/migration.php
View file

@ -22,7 +22,7 @@ $config['migration_enabled'] = TRUE;
|
*/
$config['migration_version'] = 246;
$config['migration_version'] = 247;
/*
|--------------------------------------------------------------------------

138
application/controllers/Lotw.php
View file

@ -526,6 +526,10 @@ class Lotw extends CI_Controller {
$tableheaders .= "</tr>";
$table = "";
$batch_updates = array(); // Collect all updates for batch processing
$table_rows = array(); // Collect table rows for later rendering
// First pass: collect all records for batch processing
while($record = $this->adif_parser->get_record())
{
@ -548,75 +552,93 @@ class Lotw extends CI_Controller {
$status = $this->logbook_model->import_check($time_on, $record['call'], $record['band'], $record['mode'], $record['station_callsign']);
if($status[0] == "Found") {
if (isset($record['state'])) {
$state = $record['state'];
} else {
$state = "";
}
// Present only if the QSLing station specified a single valid grid square value in its station location uploaded to LoTW.
if (isset($record['gridsquare'])) {
$qsl_gridsquare = $record['gridsquare'];
} else {
$qsl_gridsquare = "";
}
$state = isset($record['state']) ? $record['state'] : "";
$qsl_gridsquare = isset($record['gridsquare']) ? $record['gridsquare'] : "";
$qsl_vucc_grids = isset($record['vucc_grids']) ? $record['vucc_grids'] : "";
$iota = isset($record['iota']) ? $record['iota'] : "";
$cnty = isset($record['cnty']) ? $record['cnty'] : "";
$cqz = isset($record['cqz']) ? $record['cqz'] : "";
$ituz = isset($record['ituz']) ? $record['ituz'] : "";
if (isset($record['vucc_grids'])) {
$qsl_vucc_grids = $record['vucc_grids'];
} else {
$qsl_vucc_grids = "";
}
if (isset($record['iota'])) {
$iota = $record['iota'];
} else {
$iota = "";
}
if (isset($record['cnty'])) {
$cnty = $record['cnty'];
} else {
$cnty = "";
}
if (isset($record['cqz'])) {
$cqz = $record['cqz'];
} else {
$cqz = "";
}
if (isset($record['ituz'])) {
$ituz = $record['ituz'];
} else {
$ituz = "";
}
$lotw_status = $this->logbook_model->lotw_update($time_on, $record['call'], $record['band'], $qsl_date, $record['qsl_rcvd'], $state, $qsl_gridsquare, $qsl_vucc_grids, $iota, $cnty, $cqz, $ituz, $record['station_callsign']);
// Add to batch update array
$batch_updates[] = array(
'datetime' => $time_on,
'callsign' => $record['call'],
'band' => $record['band'],
'qsl_date' => $qsl_date,
'qsl_status' => $record['qsl_rcvd'],
'state' => $state,
'qsl_gridsquare' => $qsl_gridsquare,
'qsl_vucc_grids' => $qsl_vucc_grids,
'iota' => $iota,
'cnty' => $cnty,
'cqz' => $cqz,
'ituz' => $ituz,
'station_callsign' => $record['station_callsign']
);
$table_rows[] = array(
'station_callsign' => $record['station_callsign'],
'time_on' => $time_on,
'call' => $record['call'],
'mode' => $record['mode'],
'qsl_rcvd' => $record['qsl_rcvd'],
'qsl_date' => $qsl_date,
'state' => $state,
'gridsquare' => ($qsl_gridsquare != '' ? $qsl_gridsquare : $qsl_vucc_grids),
'iota' => $iota,
'status' => $status[0],
'found' => true
);
} else {
$table_rows[] = array(
'station_callsign' => $record['station_callsign'],
'time_on' => $time_on,
'call' => $record['call'],
'mode' => $record['mode'],
'qsl_rcvd' => $record['qsl_rcvd'],
'status' => $status[0],
'found' => false
);
}
}
// Batch update all LOTW confirmations in one operation
$lotw_status = "No updates";
if (!empty($batch_updates)) {
$result = $this->logbook_model->lotw_update_batch($batch_updates);
$lotw_status = "Batch Updated: {$result['updated']} QSOs, {$result['gridsquare_updated']} gridsquares";
log_message('info', 'LoTW Download: ' . $lotw_status);
}
// Build table from collected rows
foreach ($table_rows as $row) {
if ($row['found']) {
$table .= "<tr>";
$table .= "<td>".$record['station_callsign']."</td>";
$table .= "<td>".$time_on."</td>";
$table .= "<td>".$record['call']."</td>";
$table .= "<td>".$record['mode']."</td>";
$table .= "<td>".$record['qsl_rcvd']."</td>";
$table .= "<td>".$qsl_date."</td>";
$table .= "<td>".$state."</td>";
$table .= "<td>".($qsl_gridsquare != '' ? $qsl_gridsquare : $qsl_vucc_grids)."</td>";
$table .= "<td>".$iota."</td>";
$table .= "<td>QSO Record: ".$status[0]."</td>";
$table .= "<td>".$row['station_callsign']."</td>";
$table .= "<td>".$row['time_on']."</td>";
$table .= "<td>".$row['call']."</td>";
$table .= "<td>".$row['mode']."</td>";
$table .= "<td>".$row['qsl_rcvd']."</td>";
$table .= "<td>".$row['qsl_date']."</td>";
$table .= "<td>".$row['state']."</td>";
$table .= "<td>".$row['gridsquare']."</td>";
$table .= "<td>".$row['iota']."</td>";
$table .= "<td>QSO Record: ".$row['status']."</td>";
$table .= "<td>LoTW Record: ".$lotw_status."</td>";
$table .= "</tr>";
} else {
$table .= "<tr>";
$table .= "<td>".$record['station_callsign']."</td>";
$table .= "<td>".$time_on."</td>";
$table .= "<td>".$record['call']."</td>";
$table .= "<td>".$record['mode']."</td>";
$table .= "<td>".$record['qsl_rcvd']."</td>";
$table .= "<td>".$row['station_callsign']."</td>";
$table .= "<td>".$row['time_on']."</td>";
$table .= "<td>".$row['call']."</td>";
$table .= "<td>".$row['mode']."</td>";
$table .= "<td>".$row['qsl_rcvd']."</td>";
$table .= "<td></td>";
$table .= "<td></td>";
$table .= "<td></td>";
$table .= "<td></td>";
$table .= "<td>QSO Record: ".$status[0]."</td>";
$table .= "<td>QSO Record: ".$row['status']."</td>";
$table .= "<td></td>";
$table .= "</tr>";
}

78
application/migrations/247_add_lotw_performance_indexes.php Normal file
View file

@ -0,0 +1,78 @@
<?php
defined('BASEPATH') or exit('No direct script access allowed');
class Migration_add_lotw_performance_indexes extends CI_Migration
{
public function up()
{
$this->db->db_debug = false;
$table_name = $this->config->item('table_name');
// Add index on COL_LOTW_QSLRDATE for lotw_last_qsl_date() query
// Dramatically improves performance when finding the last LOTW confirmation date
$lotw_qslrdate_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_qslrdate'")->num_rows();
if ($lotw_qslrdate_index_exists == 0) {
$sql = "ALTER TABLE $table_name ADD INDEX `idx_lotw_qslrdate` (`COL_LOTW_QSLRDATE`)";
$this->db->query($sql);
}
// Add composite index for get_lotw_qsos_to_upload() query
// This index covers the WHERE and ORDER BY clauses for efficient LOTW upload preparation
// Format: (station_id, COL_LOTW_QSL_SENT, COL_TIME_ON)
$lotw_upload_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_qsos_to_upload'")->num_rows();
if ($lotw_upload_index_exists == 0) {
$sql = "ALTER TABLE $table_name ADD INDEX `idx_lotw_qsos_to_upload` (`station_id`, `COL_LOTW_QSL_SENT`, `COL_TIME_ON`)";
$this->db->query($sql);
}
// Add composite index for LOTW confirmation matching in batch downloads
// Speeds up import_check() and lotw_update_batch() lookups
// Format: (COL_TIME_ON, COL_CALL, COL_BAND, COL_STATION_CALLSIGN)
$lotw_match_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_confirmation_match'")->num_rows();
if ($lotw_match_index_exists == 0) {
$sql = "ALTER TABLE $table_name ADD INDEX `idx_lotw_confirmation_match` (`COL_TIME_ON`, `COL_CALL`, `COL_BAND`, `COL_STATION_CALLSIGN`)";
$this->db->query($sql);
}
$this->db->db_debug = true;
}
public function down()
{
$this->db->db_debug = false;
$table_name = $this->config->item('table_name');
// Drop the LOTW QSLRDATE index if it exists
$lotw_qslrdate_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_qslrdate'")->num_rows();
if ($lotw_qslrdate_index_exists > 0) {
$sql = "ALTER TABLE $table_name DROP INDEX `idx_lotw_qslrdate`";
$this->db->query($sql);
}
// Drop the LOTW upload index if it exists
$lotw_upload_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_qsos_to_upload'")->num_rows();
if ($lotw_upload_index_exists > 0) {
$sql = "ALTER TABLE $table_name DROP INDEX `idx_lotw_qsos_to_upload`";
$this->db->query($sql);
}
// Drop the LOTW confirmation match index if it exists
$lotw_match_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_confirmation_match'")->num_rows();
if ($lotw_match_index_exists > 0) {
$sql = "ALTER TABLE $table_name DROP INDEX `idx_lotw_confirmation_match`";
$this->db->query($sql);
}
$this->db->db_debug = true;
}
}

176
application/models/Logbook_model.php
View file

@ -3504,6 +3504,182 @@ class Logbook_model extends CI_Model
return "Updated";
}
/*
* Batch update multiple QSOs from LoTW confirmation downloads
* Replaces N individual lotw_update() calls with batch operations
* Provides massive performance improvement for large LOTW downloads
*
* @param array $records Array of LOTW confirmation records with keys:
* datetime, callsign, band, qsl_date, qsl_status,
* state, qsl_gridsquare, qsl_vucc_grids, iota,
* cnty, cqz, ituz, station_callsign
* @return array Statistics array with 'updated', 'gridsquare_updated', 'errors' counts
*/
function lotw_update_batch($records)
{
if (empty($records) || !is_array($records)) {
log_message('debug', 'LoTW batch update: No records provided');
return array('updated' => 0, 'gridsquare_updated' => 0, 'errors' => 0);
}
$record_count = count($records);
log_message('info', "LoTW batch update: Processing {$record_count} confirmation records");
$table_name = $this->config->item('table_name');
// Step 1: Build WHERE conditions to find all matching QSOs and their current upload status
$match_conditions = array();
foreach ($records as $idx => $record) {
$match_conditions[] = sprintf(
"(date_format(COL_TIME_ON, '%%Y-%%m-%%d %%H:%%i') = '%s' AND COL_CALL = '%s' AND COL_BAND = '%s' AND COL_STATION_CALLSIGN = '%s')",
$this->db->escape_str($record['datetime']),
$this->db->escape_str($record['callsign']),
$this->db->escape_str($record['band']),
$this->db->escape_str($record['station_callsign'])
);
}
// Step 2: Get all matching QSOs with their current upload status and gridsquare
$sql = "SELECT COL_PRIMARY_KEY,
date_format(COL_TIME_ON, '%Y-%m-%d %H:%i') as fmt_time,
COL_CALL, COL_BAND, COL_STATION_CALLSIGN,
COL_CLUBLOG_QSO_UPLOAD_STATUS as CL_STATE,
COL_QRZCOM_QSO_UPLOAD_STATUS as QRZ_STATE,
COL_LOTW_QSL_RCVD,
station_profile.station_gridsquare,
station_profile.station_id
FROM {$table_name}
LEFT JOIN station_profile ON {$table_name}.station_id = station_profile.station_id
WHERE (" . implode(' OR ', $match_conditions) . ")";
$query = $this->db->query($sql);
if ($query->num_rows() == 0) {
log_message('warning', 'LoTW batch update: No matching QSOs found in database');
return array('updated' => 0, 'gridsquare_updated' => 0, 'errors' => 0);
}
// Step 3: Build lookup map and batch update array
$qso_map = array();
foreach ($query->result() as $qso) {
$key = $qso->fmt_time . '|' . $qso->COL_CALL . '|' . $qso->COL_BAND . '|' . $qso->COL_STATION_CALLSIGN;
$qso_map[$key] = $qso;
}
$batch_updates = array();
$gridsquare_updates = array();
foreach ($records as $record) {
$key = $record['datetime'] . '|' . $record['callsign'] . '|' . $record['band'] . '|' . $record['station_callsign'];
if (!isset($qso_map[$key])) {
continue; // QSO not found in database
}
$qso = $qso_map[$key];
// Skip if already updated with this exact status
if ($qso->COL_LOTW_QSL_RCVD == $record['qsl_status']) {
continue;
}
$update_data = array(
'COL_PRIMARY_KEY' => $qso->COL_PRIMARY_KEY,
'COL_LOTW_QSLRDATE' => $record['qsl_date'],
'COL_LOTW_QSL_RCVD' => $record['qsl_status'],
'COL_LOTW_QSL_SENT' => 'Y'
);
// Add optional fields
if (!empty($record['state'])) {
$update_data['COL_STATE'] = $record['state'];
}
if (!empty($record['iota'])) {
$update_data['COL_IOTA'] = $record['iota'];
}
if (!empty($record['cnty'])) {
$update_data['COL_CNTY'] = $record['cnty'];
}
if (!empty($record['cqz'])) {
$update_data['COL_CQZ'] = $record['cqz'];
}
if (!empty($record['ituz'])) {
$update_data['COL_ITUZ'] = $record['ituz'];
}
// Mark for re-upload to QRZ/ClubLog if already uploaded
if ($qso->QRZ_STATE == 'Y') {
$update_data['COL_QRZCOM_QSO_UPLOAD_STATUS'] = 'M';
}
if ($qso->CL_STATE == 'Y') {
$update_data['COL_CLUBLOG_QSO_UPLOAD_STATUS'] = 'M';
}
$batch_updates[] = $update_data;
// Handle gridsquare updates separately (need distance calculation)
if (!empty($record['qsl_gridsquare']) || !empty($record['qsl_vucc_grids'])) {
$gridsquare_updates[] = array(
'COL_PRIMARY_KEY' => $qso->COL_PRIMARY_KEY,
'station_gridsquare' => $qso->station_gridsquare,
'qsl_gridsquare' => $record['qsl_gridsquare'] ?? '',
'qsl_vucc_grids' => $record['qsl_vucc_grids'] ?? ''
);
}
}
// Step 4: Execute batch update
$updated_count = 0;
if (!empty($batch_updates)) {
$this->db->update_batch($table_name, $batch_updates, 'COL_PRIMARY_KEY');
$updated_count = $this->db->affected_rows();
log_message('info', "LoTW batch update: Updated {$updated_count} QSO records");
}
// Step 5: Handle gridsquare updates with distance calculation
$gridsquare_count = 0;
if (!empty($gridsquare_updates)) {
if (!$this->load->is_loaded('Qra')) {
$this->load->library('Qra');
}
$grid_batch = array();
foreach ($gridsquare_updates as $grid_update) {
$data = array('COL_PRIMARY_KEY' => $grid_update['COL_PRIMARY_KEY']);
if (!empty($grid_update['qsl_gridsquare'])) {
$data['COL_GRIDSQUARE'] = $grid_update['qsl_gridsquare'];
$data['COL_DISTANCE'] = $this->qra->distance(
$grid_update['station_gridsquare'],
$grid_update['qsl_gridsquare'],
'K'
);
} elseif (!empty($grid_update['qsl_vucc_grids'])) {
$data['COL_VUCC_GRIDS'] = $grid_update['qsl_vucc_grids'];
$data['COL_DISTANCE'] = $this->qra->distance(
$grid_update['station_gridsquare'],
$grid_update['qsl_vucc_grids'],
'K'
);
}
$grid_batch[] = $data;
}
if (!empty($grid_batch)) {
$this->db->update_batch($table_name, $grid_batch, 'COL_PRIMARY_KEY');
$gridsquare_count = $this->db->affected_rows();
log_message('info', "LoTW batch update: Updated {$gridsquare_count} gridsquare/distance records");
}
}
return array(
'updated' => $updated_count,
'gridsquare_updated' => $gridsquare_count,
'errors' => 0
);
}
function qrz_last_qsl_date($user_id)
{
$sql = "SELECT date_format(MAX(COALESCE(COL_QRZCOM_QSO_DOWNLOAD_DATE, str_to_date('1900-01-01','%Y-%m-%d'))),'%Y-%m-%d') MAXDATE

28
application/models/LotwCert.php
View file

@ -78,26 +78,28 @@ class LotwCert extends CI_Model {
}
function toggle_archive_certificate($user_id, $lotw_cert_id) {
// First get current archive status
// Use single UPDATE with calculated toggle instead of SELECT+UPDATE
// archived = 1 - archived toggles between 0 and 1
$this->db->set('archived', '1 - archived', FALSE);
$this->db->where('lotw_cert_id', $lotw_cert_id);
$this->db->where('user_id', $user_id);
$this->db->update('lotw_certs');
if ($this->db->affected_rows() == 0) {
return false;
}
// Get final status to return
$this->db->select('archived');
$this->db->where('lotw_cert_id', $lotw_cert_id);
$this->db->where('user_id', $user_id);
$query = $this->db->get('lotw_certs');
if($query->num_rows() == 0) {
return false;
if($query->num_rows() > 0) {
return array('archived' => $query->row()->archived);
}
$current_status = $query->row()->archived;
$new_status = $current_status ? 0 : 1;
// Update the archive status
$data = array('archived' => $new_status);
$this->db->where('lotw_cert_id', $lotw_cert_id);
$this->db->where('user_id', $user_id);
$this->db->update('lotw_certs', $data);
return array('archived' => $new_status);
return false;
}
function last_upload($certID) {