diff --git a/application/config/migration.php b/application/config/migration.php
index 343f22cd0..9948fe947 100644
--- a/application/config/migration.php
+++ b/application/config/migration.php
@@ -22,7 +22,7 @@ $config['migration_enabled'] = TRUE;
|
*/
-$config['migration_version'] = 246;
+$config['migration_version'] = 247;
/*
|--------------------------------------------------------------------------
diff --git a/application/controllers/Lotw.php b/application/controllers/Lotw.php
index 1a0aaf37c..56cb66215 100644
--- a/application/controllers/Lotw.php
+++ b/application/controllers/Lotw.php
@@ -526,6 +526,10 @@ class Lotw extends CI_Controller {
$tableheaders .= "";
$table = "";
+ $batch_updates = array(); // Collect all updates for batch processing
+ $table_rows = array(); // Collect table rows for later rendering
+
+ // First pass: collect all records for batch processing
while($record = $this->adif_parser->get_record())
{
@@ -548,75 +552,93 @@ class Lotw extends CI_Controller {
$status = $this->logbook_model->import_check($time_on, $record['call'], $record['band'], $record['mode'], $record['station_callsign']);
if($status[0] == "Found") {
- if (isset($record['state'])) {
- $state = $record['state'];
- } else {
- $state = "";
- }
- // Present only if the QSLing station specified a single valid grid square value in its station location uploaded to LoTW.
- if (isset($record['gridsquare'])) {
- $qsl_gridsquare = $record['gridsquare'];
- } else {
- $qsl_gridsquare = "";
- }
+ $state = isset($record['state']) ? $record['state'] : "";
+ $qsl_gridsquare = isset($record['gridsquare']) ? $record['gridsquare'] : "";
+ $qsl_vucc_grids = isset($record['vucc_grids']) ? $record['vucc_grids'] : "";
+ $iota = isset($record['iota']) ? $record['iota'] : "";
+ $cnty = isset($record['cnty']) ? $record['cnty'] : "";
+ $cqz = isset($record['cqz']) ? $record['cqz'] : "";
+ $ituz = isset($record['ituz']) ? $record['ituz'] : "";
- if (isset($record['vucc_grids'])) {
- $qsl_vucc_grids = $record['vucc_grids'];
- } else {
- $qsl_vucc_grids = "";
- }
-
- if (isset($record['iota'])) {
- $iota = $record['iota'];
- } else {
- $iota = "";
- }
-
- if (isset($record['cnty'])) {
- $cnty = $record['cnty'];
- } else {
- $cnty = "";
- }
-
- if (isset($record['cqz'])) {
- $cqz = $record['cqz'];
- } else {
- $cqz = "";
- }
-
- if (isset($record['ituz'])) {
- $ituz = $record['ituz'];
- } else {
- $ituz = "";
- }
-
- $lotw_status = $this->logbook_model->lotw_update($time_on, $record['call'], $record['band'], $qsl_date, $record['qsl_rcvd'], $state, $qsl_gridsquare, $qsl_vucc_grids, $iota, $cnty, $cqz, $ituz, $record['station_callsign']);
+ // Add to batch update array
+ $batch_updates[] = array(
+ 'datetime' => $time_on,
+ 'callsign' => $record['call'],
+ 'band' => $record['band'],
+ 'qsl_date' => $qsl_date,
+ 'qsl_status' => $record['qsl_rcvd'],
+ 'state' => $state,
+ 'qsl_gridsquare' => $qsl_gridsquare,
+ 'qsl_vucc_grids' => $qsl_vucc_grids,
+ 'iota' => $iota,
+ 'cnty' => $cnty,
+ 'cqz' => $cqz,
+ 'ituz' => $ituz,
+ 'station_callsign' => $record['station_callsign']
+ );
+ $table_rows[] = array(
+ 'station_callsign' => $record['station_callsign'],
+ 'time_on' => $time_on,
+ 'call' => $record['call'],
+ 'mode' => $record['mode'],
+ 'qsl_rcvd' => $record['qsl_rcvd'],
+ 'qsl_date' => $qsl_date,
+ 'state' => $state,
+ 'gridsquare' => ($qsl_gridsquare != '' ? $qsl_gridsquare : $qsl_vucc_grids),
+ 'iota' => $iota,
+ 'status' => $status[0],
+ 'found' => true
+ );
+ } else {
+ $table_rows[] = array(
+ 'station_callsign' => $record['station_callsign'],
+ 'time_on' => $time_on,
+ 'call' => $record['call'],
+ 'mode' => $record['mode'],
+ 'qsl_rcvd' => $record['qsl_rcvd'],
+ 'status' => $status[0],
+ 'found' => false
+ );
+ }
+ }
+
+ // Batch update all LOTW confirmations in one operation
+ $lotw_status = "No updates";
+ if (!empty($batch_updates)) {
+ $result = $this->logbook_model->lotw_update_batch($batch_updates);
+ $lotw_status = "Batch Updated: {$result['updated']} QSOs, {$result['gridsquare_updated']} gridsquares";
+ log_message('info', 'LoTW Download: ' . $lotw_status);
+ }
+
+ // Build table from collected rows
+ foreach ($table_rows as $row) {
+ if ($row['found']) {
$table .= "
";
- $table .= "| ".$record['station_callsign']." | ";
- $table .= "".$time_on." | ";
- $table .= "".$record['call']." | ";
- $table .= "".$record['mode']." | ";
- $table .= "".$record['qsl_rcvd']." | ";
- $table .= "".$qsl_date." | ";
- $table .= "".$state." | ";
- $table .= "".($qsl_gridsquare != '' ? $qsl_gridsquare : $qsl_vucc_grids)." | ";
- $table .= "".$iota." | ";
- $table .= "QSO Record: ".$status[0]." | ";
+ $table .= "".$row['station_callsign']." | ";
+ $table .= "".$row['time_on']." | ";
+ $table .= "".$row['call']." | ";
+ $table .= "".$row['mode']." | ";
+ $table .= "".$row['qsl_rcvd']." | ";
+ $table .= "".$row['qsl_date']." | ";
+ $table .= "".$row['state']." | ";
+ $table .= "".$row['gridsquare']." | ";
+ $table .= "".$row['iota']." | ";
+ $table .= "QSO Record: ".$row['status']." | ";
$table .= "LoTW Record: ".$lotw_status." | ";
$table .= "
";
} else {
$table .= "";
- $table .= "| ".$record['station_callsign']." | ";
- $table .= "".$time_on." | ";
- $table .= "".$record['call']." | ";
- $table .= "".$record['mode']." | ";
- $table .= "".$record['qsl_rcvd']." | ";
+ $table .= "".$row['station_callsign']." | ";
+ $table .= "".$row['time_on']." | ";
+ $table .= "".$row['call']." | ";
+ $table .= "".$row['mode']." | ";
+ $table .= "".$row['qsl_rcvd']." | ";
$table .= " | ";
$table .= " | ";
$table .= " | ";
$table .= " | ";
- $table .= "QSO Record: ".$status[0]." | ";
+ $table .= "QSO Record: ".$row['status']." | ";
$table .= " | ";
$table .= "
";
}
diff --git a/application/migrations/247_add_lotw_performance_indexes.php b/application/migrations/247_add_lotw_performance_indexes.php
new file mode 100644
index 000000000..4cd471862
--- /dev/null
+++ b/application/migrations/247_add_lotw_performance_indexes.php
@@ -0,0 +1,78 @@
+db->db_debug = false;
+
+ $table_name = $this->config->item('table_name');
+
+ // Add index on COL_LOTW_QSLRDATE for lotw_last_qsl_date() query
+ // Dramatically improves performance when finding the last LOTW confirmation date
+ $lotw_qslrdate_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_qslrdate'")->num_rows();
+
+ if ($lotw_qslrdate_index_exists == 0) {
+ $sql = "ALTER TABLE $table_name ADD INDEX `idx_lotw_qslrdate` (`COL_LOTW_QSLRDATE`)";
+ $this->db->query($sql);
+ }
+
+ // Add composite index for get_lotw_qsos_to_upload() query
+ // This index covers the WHERE and ORDER BY clauses for efficient LOTW upload preparation
+ // Format: (station_id, COL_LOTW_QSL_SENT, COL_TIME_ON)
+ $lotw_upload_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_qsos_to_upload'")->num_rows();
+
+ if ($lotw_upload_index_exists == 0) {
+ $sql = "ALTER TABLE $table_name ADD INDEX `idx_lotw_qsos_to_upload` (`station_id`, `COL_LOTW_QSL_SENT`, `COL_TIME_ON`)";
+ $this->db->query($sql);
+ }
+
+ // Add composite index for LOTW confirmation matching in batch downloads
+ // Speeds up import_check() and lotw_update_batch() lookups
+ // Format: (COL_TIME_ON, COL_CALL, COL_BAND, COL_STATION_CALLSIGN)
+ $lotw_match_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_confirmation_match'")->num_rows();
+
+ if ($lotw_match_index_exists == 0) {
+ $sql = "ALTER TABLE $table_name ADD INDEX `idx_lotw_confirmation_match` (`COL_TIME_ON`, `COL_CALL`, `COL_BAND`, `COL_STATION_CALLSIGN`)";
+ $this->db->query($sql);
+ }
+
+ $this->db->db_debug = true;
+ }
+
+ public function down()
+ {
+ $this->db->db_debug = false;
+
+ $table_name = $this->config->item('table_name');
+
+ // Drop the LOTW QSLRDATE index if it exists
+ $lotw_qslrdate_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_qslrdate'")->num_rows();
+
+ if ($lotw_qslrdate_index_exists > 0) {
+ $sql = "ALTER TABLE $table_name DROP INDEX `idx_lotw_qslrdate`";
+ $this->db->query($sql);
+ }
+
+ // Drop the LOTW upload index if it exists
+ $lotw_upload_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_qsos_to_upload'")->num_rows();
+
+ if ($lotw_upload_index_exists > 0) {
+ $sql = "ALTER TABLE $table_name DROP INDEX `idx_lotw_qsos_to_upload`";
+ $this->db->query($sql);
+ }
+
+ // Drop the LOTW confirmation match index if it exists
+ $lotw_match_index_exists = $this->db->query("SHOW INDEX FROM $table_name WHERE Key_name = 'idx_lotw_confirmation_match'")->num_rows();
+
+ if ($lotw_match_index_exists > 0) {
+ $sql = "ALTER TABLE $table_name DROP INDEX `idx_lotw_confirmation_match`";
+ $this->db->query($sql);
+ }
+
+ $this->db->db_debug = true;
+ }
+}
diff --git a/application/models/Logbook_model.php b/application/models/Logbook_model.php
index ab0b625dc..5237aaf68 100755
--- a/application/models/Logbook_model.php
+++ b/application/models/Logbook_model.php
@@ -3504,6 +3504,182 @@ class Logbook_model extends CI_Model
return "Updated";
}
+ /*
+ * Batch update multiple QSOs from LoTW confirmation downloads
+ * Replaces N individual lotw_update() calls with batch operations
+ * Provides massive performance improvement for large LOTW downloads
+ *
+ * @param array $records Array of LOTW confirmation records with keys:
+ * datetime, callsign, band, qsl_date, qsl_status,
+ * state, qsl_gridsquare, qsl_vucc_grids, iota,
+ * cnty, cqz, ituz, station_callsign
+ * @return array Statistics array with 'updated', 'gridsquare_updated', 'errors' counts
+ */
+ function lotw_update_batch($records)
+ {
+ if (empty($records) || !is_array($records)) {
+ log_message('debug', 'LoTW batch update: No records provided');
+ return array('updated' => 0, 'gridsquare_updated' => 0, 'errors' => 0);
+ }
+
+ $record_count = count($records);
+ log_message('info', "LoTW batch update: Processing {$record_count} confirmation records");
+
+ $table_name = $this->config->item('table_name');
+
+ // Step 1: Build WHERE conditions to find all matching QSOs and their current upload status
+ $match_conditions = array();
+ foreach ($records as $idx => $record) {
+ $match_conditions[] = sprintf(
+ "(date_format(COL_TIME_ON, '%%Y-%%m-%%d %%H:%%i') = '%s' AND COL_CALL = '%s' AND COL_BAND = '%s' AND COL_STATION_CALLSIGN = '%s')",
+ $this->db->escape_str($record['datetime']),
+ $this->db->escape_str($record['callsign']),
+ $this->db->escape_str($record['band']),
+ $this->db->escape_str($record['station_callsign'])
+ );
+ }
+
+ // Step 2: Get all matching QSOs with their current upload status and gridsquare
+ $sql = "SELECT COL_PRIMARY_KEY,
+ date_format(COL_TIME_ON, '%Y-%m-%d %H:%i') as fmt_time,
+ COL_CALL, COL_BAND, COL_STATION_CALLSIGN,
+ COL_CLUBLOG_QSO_UPLOAD_STATUS as CL_STATE,
+ COL_QRZCOM_QSO_UPLOAD_STATUS as QRZ_STATE,
+ COL_LOTW_QSL_RCVD,
+ station_profile.station_gridsquare,
+ station_profile.station_id
+ FROM {$table_name}
+ LEFT JOIN station_profile ON {$table_name}.station_id = station_profile.station_id
+ WHERE (" . implode(' OR ', $match_conditions) . ")";
+
+ $query = $this->db->query($sql);
+
+ if ($query->num_rows() == 0) {
+ log_message('warning', 'LoTW batch update: No matching QSOs found in database');
+ return array('updated' => 0, 'gridsquare_updated' => 0, 'errors' => 0);
+ }
+
+ // Step 3: Build lookup map and batch update array
+ $qso_map = array();
+ foreach ($query->result() as $qso) {
+ $key = $qso->fmt_time . '|' . $qso->COL_CALL . '|' . $qso->COL_BAND . '|' . $qso->COL_STATION_CALLSIGN;
+ $qso_map[$key] = $qso;
+ }
+
+ $batch_updates = array();
+ $gridsquare_updates = array();
+
+ foreach ($records as $record) {
+ $key = $record['datetime'] . '|' . $record['callsign'] . '|' . $record['band'] . '|' . $record['station_callsign'];
+
+ if (!isset($qso_map[$key])) {
+ continue; // QSO not found in database
+ }
+
+ $qso = $qso_map[$key];
+
+ // Skip if already updated with this exact status
+ if ($qso->COL_LOTW_QSL_RCVD == $record['qsl_status']) {
+ continue;
+ }
+
+ $update_data = array(
+ 'COL_PRIMARY_KEY' => $qso->COL_PRIMARY_KEY,
+ 'COL_LOTW_QSLRDATE' => $record['qsl_date'],
+ 'COL_LOTW_QSL_RCVD' => $record['qsl_status'],
+ 'COL_LOTW_QSL_SENT' => 'Y'
+ );
+
+ // Add optional fields
+ if (!empty($record['state'])) {
+ $update_data['COL_STATE'] = $record['state'];
+ }
+ if (!empty($record['iota'])) {
+ $update_data['COL_IOTA'] = $record['iota'];
+ }
+ if (!empty($record['cnty'])) {
+ $update_data['COL_CNTY'] = $record['cnty'];
+ }
+ if (!empty($record['cqz'])) {
+ $update_data['COL_CQZ'] = $record['cqz'];
+ }
+ if (!empty($record['ituz'])) {
+ $update_data['COL_ITUZ'] = $record['ituz'];
+ }
+
+ // Mark for re-upload to QRZ/ClubLog if already uploaded
+ if ($qso->QRZ_STATE == 'Y') {
+ $update_data['COL_QRZCOM_QSO_UPLOAD_STATUS'] = 'M';
+ }
+ if ($qso->CL_STATE == 'Y') {
+ $update_data['COL_CLUBLOG_QSO_UPLOAD_STATUS'] = 'M';
+ }
+
+ $batch_updates[] = $update_data;
+
+ // Handle gridsquare updates separately (need distance calculation)
+ if (!empty($record['qsl_gridsquare']) || !empty($record['qsl_vucc_grids'])) {
+ $gridsquare_updates[] = array(
+ 'COL_PRIMARY_KEY' => $qso->COL_PRIMARY_KEY,
+ 'station_gridsquare' => $qso->station_gridsquare,
+ 'qsl_gridsquare' => $record['qsl_gridsquare'] ?? '',
+ 'qsl_vucc_grids' => $record['qsl_vucc_grids'] ?? ''
+ );
+ }
+ }
+
+ // Step 4: Execute batch update
+ $updated_count = 0;
+ if (!empty($batch_updates)) {
+ $this->db->update_batch($table_name, $batch_updates, 'COL_PRIMARY_KEY');
+ $updated_count = $this->db->affected_rows();
+ log_message('info', "LoTW batch update: Updated {$updated_count} QSO records");
+ }
+
+ // Step 5: Handle gridsquare updates with distance calculation
+ $gridsquare_count = 0;
+ if (!empty($gridsquare_updates)) {
+ if (!$this->load->is_loaded('Qra')) {
+ $this->load->library('Qra');
+ }
+
+ $grid_batch = array();
+ foreach ($gridsquare_updates as $grid_update) {
+ $data = array('COL_PRIMARY_KEY' => $grid_update['COL_PRIMARY_KEY']);
+
+ if (!empty($grid_update['qsl_gridsquare'])) {
+ $data['COL_GRIDSQUARE'] = $grid_update['qsl_gridsquare'];
+ $data['COL_DISTANCE'] = $this->qra->distance(
+ $grid_update['station_gridsquare'],
+ $grid_update['qsl_gridsquare'],
+ 'K'
+ );
+ } elseif (!empty($grid_update['qsl_vucc_grids'])) {
+ $data['COL_VUCC_GRIDS'] = $grid_update['qsl_vucc_grids'];
+ $data['COL_DISTANCE'] = $this->qra->distance(
+ $grid_update['station_gridsquare'],
+ $grid_update['qsl_vucc_grids'],
+ 'K'
+ );
+ }
+
+ $grid_batch[] = $data;
+ }
+
+ if (!empty($grid_batch)) {
+ $this->db->update_batch($table_name, $grid_batch, 'COL_PRIMARY_KEY');
+ $gridsquare_count = $this->db->affected_rows();
+ log_message('info', "LoTW batch update: Updated {$gridsquare_count} gridsquare/distance records");
+ }
+ }
+
+ return array(
+ 'updated' => $updated_count,
+ 'gridsquare_updated' => $gridsquare_count,
+ 'errors' => 0
+ );
+ }
+
function qrz_last_qsl_date($user_id)
{
$sql = "SELECT date_format(MAX(COALESCE(COL_QRZCOM_QSO_DOWNLOAD_DATE, str_to_date('1900-01-01','%Y-%m-%d'))),'%Y-%m-%d') MAXDATE
diff --git a/application/models/LotwCert.php b/application/models/LotwCert.php
index 0c97fc699..7d49ac0c9 100644
--- a/application/models/LotwCert.php
+++ b/application/models/LotwCert.php
@@ -78,26 +78,28 @@ class LotwCert extends CI_Model {
}
function toggle_archive_certificate($user_id, $lotw_cert_id) {
- // First get current archive status
+ // Use single UPDATE with calculated toggle instead of SELECT+UPDATE
+ // archived = 1 - archived toggles between 0 and 1
+ $this->db->set('archived', '1 - archived', FALSE);
+ $this->db->where('lotw_cert_id', $lotw_cert_id);
+ $this->db->where('user_id', $user_id);
+ $this->db->update('lotw_certs');
+
+ if ($this->db->affected_rows() == 0) {
+ return false;
+ }
+
+ // Get final status to return
$this->db->select('archived');
$this->db->where('lotw_cert_id', $lotw_cert_id);
$this->db->where('user_id', $user_id);
$query = $this->db->get('lotw_certs');
- if($query->num_rows() == 0) {
- return false;
+ if($query->num_rows() > 0) {
+ return array('archived' => $query->row()->archived);
}
- $current_status = $query->row()->archived;
- $new_status = $current_status ? 0 : 1;
-
- // Update the archive status
- $data = array('archived' => $new_status);
- $this->db->where('lotw_cert_id', $lotw_cert_id);
- $this->db->where('user_id', $user_id);
- $this->db->update('lotw_certs', $data);
-
- return array('archived' => $new_status);
+ return false;
}
function last_upload($certID) {