cloudlog/application/controllers/Update.php
Peter Goodhall 897ba50c0e Add UI for manual data file updates in maintenance
Introduces a new section in the maintenance page allowing manual updates of reference data files (Clublog SCP, LoTW Users, DOK, SOTA, WWFF, POTA) via AJAX buttons with loading indicators. Adds corresponding UI-friendly controller methods with authorization checks and improves the WWFF update logic to process CSV files line by line for better memory efficiency.
2026-01-14 14:55:07 +00:00

710 lines
23 KiB
PHP

<?php
class Update extends CI_Controller {
/*
Controls Updating Elements of Cloudlog
Functions:
dxcc - imports the latest clublog cty.xml data
lotw_users - imports lotw users
*/
public function index()
{
$data['page_title'] = "Updates";
$this->load->view('interface_assets/header', $data);
$this->load->view('update/index');
$this->load->view('interface_assets/footer');
}
/*
* Create a path to a file in the updates folder, respecting the datadir
* configuration option.
*/
private function make_update_path($path) {
$path = "updates/" . $path;
$datadir = $this->config->item('datadir');
if(!$datadir) {
return $path;
}
return $datadir . "/" . $path;
}
/*
* Load the dxcc entities
*/
public function dxcc_entities() {
// Load Database connectors
$this->load->model('dxcc_entities');
// Load the cty file
$xml_data = simplexml_load_file($this->make_update_path("cty.xml"));
//$xml_data->entities->entity->count();
$count = 0;
foreach ($xml_data->entities->entity as $entity) {
$startinfo = strtotime($entity->start);
$endinfo = strtotime($entity->end);
$start_date = ($startinfo) ? date('Y-m-d H:i:s',$startinfo) : null;
$end_date = ($endinfo) ? date('Y-m-d H:i:s',$endinfo) : null;
if(!$entity->cqz) {
$data = array(
'prefix' => (string) $entity->call,
'name' => (string) $entity->entity,
);
} else {
$data = array(
'adif' => (int) $entity->adif,
'name' => (string) $entity->name,
'prefix' => (string) $entity->prefix,
'ituz' => (float) $entity->ituz,
'cqz' => (int) $entity->cqz,
'cont' => (string) $entity->cont,
'long' => (float) $entity->long,
'lat' => (float) $entity->lat,
'start' => $start_date,
'end' => $end_date,
);
}
$this->db->insert('dxcc_entities', $data);
$count += 1;
if ($count % 10 == 0)
$this->update_status();
}
$this->update_status();
return $count;
}
/*
* Load the dxcc exceptions
*/
public function dxcc_exceptions() {
// Load Database connectors
$this->load->model('dxcc_exceptions');
// Load the cty file
$xml_data = simplexml_load_file($this->make_update_path("cty.xml"));
$count = 0;
foreach ($xml_data->exceptions->exception as $record) {
$startinfo = strtotime($record->start);
$endinfo = strtotime($record->end);
$start_date = ($startinfo) ? date('Y-m-d H:i:s',$startinfo) : null;
$end_date = ($endinfo) ? date('Y-m-d H:i:s',$endinfo) : null;
$data = array(
'record' => (int) $record->attributes()->record,
'call' => (string) $record->call,
'entity' => (string) $record->entity,
'adif' => (int) $record->adif,
'cqz' => (int) $record->cqz,
'cont' => (string) $record->cont,
'long' => (float) $record->long,
'lat' => (float) $record->lat,
'start' => $start_date,
'end' => $end_date,
);
$this->db->insert('dxcc_exceptions', $data);
$count += 1;
if ($count % 10 == 0)
$this->update_status();
}
$this->update_status();
return $count;
}
/*
* Load the dxcc prefixes
*/
public function dxcc_prefixes() {
// Load Database connectors
$this->load->model('dxcc_prefixes');
// Load the cty file
$xml_data = simplexml_load_file($this->make_update_path("cty.xml"));
$count = 0;
foreach ($xml_data->prefixes->prefix as $record) {
$startinfo = strtotime($record->start);
$endinfo = strtotime($record->end);
$start_date = ($startinfo) ? date('Y-m-d H:i:s',$startinfo) : null;
$end_date = ($endinfo) ? date('Y-m-d H:i:s',$endinfo) : null;
$data = array(
'record' => (int) $record->attributes()->record,
'call' => (string) $record->call,
'entity' => (string) $record->entity,
'adif' => (int) $record->adif,
'cqz' => (int) $record->cqz,
'cont' => (string) $record->cont,
'long' => (float) $record->long,
'lat' => (float) $record->lat,
'start' => $start_date,
'end' => $end_date,
);
$this->db->insert('dxcc_prefixes', $data);
$count += 1;
if ($count % 10 == 0)
$this->update_status();
}
//print("$count prefixes processed");
$this->update_status();
return $count;
}
// Updates the DXCC & Exceptions from the Club Log Cty.xml file.
public function dxcc() {
$this->update_status("Downloading file");
// give it 10 minutes...
set_time_limit(600);
// Load Migration data if any.
$this->load->library('migration');
$this->fix_migrations();
$this->migration->latest();
// Download latest file.
$url = "https://cdn.clublog.org/cty.php?api=a11c3235cd74b88212ce726857056939d52372bd";
$gz = gzopen($url, 'r');
if ($gz === FALSE) {
// If the download from clublog.org fails, try cloudlog.org CDN.
$url = "https://cdn.cloudlog.org/clublogxml.gz";
$gz = gzopen($url, 'r');
// Log failure to log file
log_message('info', 'Failed to download cty.xml from clublog.org, trying cloudlog.org CDN');
if ($gz === FALSE) {
$this->update_status("FAILED: Could not download from clublog.org or cloudlog.org");
log_message('error', 'FAILED: Could not download exceptions from clublog.org or cloudlog.org');
return;
}
}
$data = "";
while (!gzeof($gz)) {
$data .= gzgetc($gz);
}
gzclose($gz);
if (file_put_contents($this->make_update_path("cty.xml"), $data) === FALSE) {
$this->update_status("FAILED: Could not write to cty.xml file");
return;
}
// Clear the tables, ready for new data
$this->db->empty_table("dxcc_entities");
$this->db->empty_table("dxcc_exceptions");
$this->db->empty_table("dxcc_prefixes");
$this->update_status();
// Parse the three sections of the file and update the tables
$this->db->trans_start();
$this->dxcc_entities();
$this->dxcc_exceptions();
$this->dxcc_prefixes();
$this->db->trans_complete();
$this->update_status("DONE");
}
public function get_status() {
$status_file = $this->make_update_path("status.html");
if (file_exists($status_file)) {
$content = file_get_contents($status_file);
echo $content;
} else {
echo "No status available";
}
}
public function update_status($done=""){
if ($done != "Downloading file"){
// Check that everything is done?
if ($done == ""){
$done = "Updating...";
}
$html = $done."<br/>";
$html .= "Dxcc Entities: ".$this->db->count_all('dxcc_entities')."<br/>";
$html .= "Dxcc Exceptions: ".$this->db->count_all('dxcc_exceptions')."<br/>";
$html .= "Dxcc Prefixes: ".$this->db->count_all('dxcc_prefixes')."<br/>";
}else{
$html = $done."....<br/>";
}
$status_file = $this->make_update_path("status.html");
if (file_put_contents($status_file, $html) === FALSE) {
log_message('error', 'Failed to write status file: ' . $status_file);
// Try to create the directory if it doesn't exist
$dir = dirname($status_file);
if (!is_dir($dir)) {
mkdir($dir, 0755, true);
file_put_contents($status_file, $html);
}
}
}
private function fix_migrations(){
$res = $this->db->query("select version from migrations");
if ($res->num_rows() >0){
$row = $res->row();
$version = $row->version;
if ($version < 7){
$this->db->query("update migrations set version=7");
}
}
}
public function check_missing_dxcc($all = false){
$this->load->model('logbook_model');
$this->logbook_model->check_missing_dxcc_id($all);
}
public function check_missing_continent() {
$this->load->model('logbook_model');
$this->logbook_model->check_missing_continent();
}
public function update_distances() {
$this->load->model('logbook_model');
$this->logbook_model->update_distances();
}
public function check_missing_grid($all = false){
$this->load->model('logbook_model');
$this->logbook_model->check_missing_grid_id($all);
}
public function update_clublog_scp() {
$strFile = $this->make_update_path("clublog_scp.txt");
$url = "https://cdn.clublog.org/clublog.scp.gz";
set_time_limit(300);
echo "Downloading Club Log SCP file...<br>";
$gz = gzopen($url, 'r');
if ($gz)
{
$data = "";
while (!gzeof($gz)) {
$data .= gzgetc($gz);
}
gzclose($gz);
if (file_put_contents($strFile, $data) !== FALSE)
{
$nCount = count(file($strFile));
if ($nCount > 0)
{
echo "DONE: " . number_format($nCount) . " callsigns loaded";
} else {
echo "FAILED: Empty file";
}
} else {
echo "FAILED: Could not write to Club Log SCP file";
}
} else {
echo "FAILED: Could not connect to Club Log";
}
}
public function download_lotw_users() {
$contents = file_get_contents('https://lotw.arrl.org/lotw-user-activity.csv', true);
if($contents === FALSE) {
echo "Something went wrong with fetching the LoTW users file.";
} else {
$file = './updates/lotw_users.csv';
if (file_put_contents($file, $contents) !== FALSE) { // Save our content to the file.
echo "LoTW User Data Saved.";
} else {
echo "FAILED: Could not write to LoTW users file";
}
}
}
public function lotw_users() {
$mtime = microtime();
$mtime = explode(" ",$mtime);
$mtime = $mtime[1] + $mtime[0];
$starttime = $mtime;
$file = 'https://lotw.arrl.org/lotw-user-activity.csv';
// First, try to download and validate the file
$context = stream_context_create([
'http' => [
'timeout' => 60, // 60 second timeout
'user_agent' => 'Cloudlog LoTW User Updater'
]
]);
$handle = fopen($file, "r", false, $context);
if ($handle === FALSE) {
echo "FAILED: Could not connect to LoTW server";
log_message('error', 'Failed to connect to LoTW user activity CSV');
return;
}
// Read and validate the first few lines to ensure we have valid data
$line_count = 0;
$sample_data = array();
while ($line_count < 5 && ($line = fgetcsv($handle, 1000, ",")) !== FALSE) {
if ($line[0]) { // Skip empty lines
$sample_data[] = $line;
$line_count++;
}
}
// Validate we have enough sample data
if ($line_count < 2) {
fclose($handle);
echo "FAILED: LoTW file appears to be empty or invalid";
log_message('error', 'LoTW user activity CSV appears empty or invalid');
return;
}
// Reset file pointer to beginning
fclose($handle);
$handle = fopen($file, "r", false, $context);
if ($handle === FALSE) {
echo "FAILED: Could not reopen LoTW file";
return;
}
// Only truncate table AFTER we've validated the remote file
$this->db->query("TRUNCATE TABLE lotw_users");
$i = 0; // raw rows read
$batch_count = 0;
// Use a map to deduplicate by callsign and keep the latest timestamp
$lotw_map = array();
$batch_size = 500; // Smaller batch size for better performance and memory usage
// Skip CSV header row
$data = fgetcsv($handle,1000,",");
while (($data = fgetcsv($handle,1000,",")) !== FALSE) {
if ($data[0] && isset($data[1]) && isset($data[2])) {
// Normalize callsign to uppercase before validation
$callsign = strtoupper($data[0]);
// Validate callsign format (basic check)
if (preg_match('/^[A-Z0-9\/]+$/', $callsign)) {
$i++;
// Compose timestamp string and compare; keep the latest per callsign
$ts = $data[1] . ' ' . $data[2];
// If we haven't seen this callsign, or this row is newer, store it
if (!isset($lotw_map[$callsign]) || strtotime($ts) > strtotime($lotw_map[$callsign])) {
$lotw_map[$callsign] = $ts;
}
}
}
}
fclose($handle);
// Insert deduplicated records in batches
if (!empty($lotw_map)) {
$lotwdata = array();
foreach ($lotw_map as $cs => $lu) {
$lotwdata[] = array('callsign' => $cs, 'lastupload' => $lu);
if (count($lotwdata) >= $batch_size) {
if (!$this->db->insert_batch('lotw_users', $lotwdata)) {
echo "FAILED: Database error during batch insert";
log_message('error', 'Database error during LoTW batch insert while inserting deduped map');
return;
}
$batch_count++;
$lotwdata = array();
}
}
// Final batch
if (!empty($lotwdata)) {
if (!$this->db->insert_batch('lotw_users', $lotwdata)) {
echo "FAILED: Database error during final batch insert";
log_message('error', 'Database error during LoTW final batch insert while inserting deduped map');
return;
}
$batch_count++;
}
}
// Verify we actually imported data
$final_count = $this->db->count_all('lotw_users');
if ($final_count == 0) {
echo "FAILED: No records were imported - possible data corruption";
log_message('error', 'LoTW import completed but no records in database');
return;
}
if ($i < 50000) { // Sanity check - LoTW should have at least 50k users
echo "WARNING: Only imported " . $i . " records - this seems unusually low";
log_message('warning', 'LoTW import record count unusually low: ' . $i);
}
$mtime = microtime();
$mtime = explode(" ",$mtime);
$mtime = $mtime[1] + $mtime[0];
$endtime = $mtime;
$totaltime = ($endtime - $starttime);
echo "This page was created in ".$totaltime." seconds <br />";
echo "Records read: " . $i . " <br/>";
echo "Unique callsigns inserted: " . $final_count . " <br/>";
}
public function lotw_check() {
$f = fopen('./updates/lotw_users.csv', "r");
$result = false;
while ($row = fgetcsv($f)) {
if ($row[0] == '2M0SQL/MM') {
$result = $row[0];
echo "found";
break;
}
}
fclose($f);
}
/*
* Used for autoupdating the DOK file which is used in the QSO entry dialog for autocompletion.
*/
public function update_dok() {
$contents = file_get_contents('https://www.df2et.de/cqrlog/dok_and_sdok.txt', true);
if($contents === FALSE) {
echo "Something went wrong with fetching the DOK file.";
} else {
$file = './assets/json/dok.txt';
if (file_put_contents($file, $contents) !== FALSE) { // Save our content to the file.
$nCount = count(file($file));
if ($nCount > 0)
{
echo "DONE: " . number_format($nCount) . " DOKs and SDOKs saved";
} else {
echo"FAILED: Empty file";
}
} else {
echo"FAILED: Could not write to dok.txt file";
}
}
}
/*
* Used for autoupdating the SOTA file which is used in the QSO entry dialog for autocompletion.
*/
public function update_sota() {
$this->load->library('sota');
$result = $this->sota->refreshFiles(true);
if (!$result['ok']) {
echo $result['message'];
return;
}
$details = [];
$details[] = number_format($result['saved_refs']) . " SOTA refs saved";
if (!empty($result['csv_saved'])) {
$details[] = "full CSV cached";
}
echo "DONE: " . implode('; ', $details);
}
/*
* Pulls the WWFF directory for autocompletion in QSO dialogs
*/
public function update_wwff() {
$csvfile = 'https://wwff.co/wwff-data/wwff_directory.csv';
$wwfffile = './assets/json/wwff.txt';
// Use stream context to fetch data without loading entire file into memory
$context = stream_context_create([
'http' => [
'timeout' => 60,
'user_agent' => 'Cloudlog Updater'
]
]);
$handle = fopen($csvfile, "r", false, $context);
if ($handle === FALSE) {
echo "Something went wrong with fetching the WWFF file";
return;
}
$wwfffilehandle = fopen($wwfffile, 'w');
if ($wwfffilehandle === FALSE) {
fclose($handle);
echo "FAILED: Could not write to wwff.txt file";
return;
}
$nCount = 0;
$isFirstLine = true;
// Process CSV line by line to avoid memory issues
while (($row = fgetcsv($handle, 1000, ",")) !== FALSE) {
if ($isFirstLine) {
$isFirstLine = false;
continue; // Skip header line
}
if (isset($row[0]) && $row[0]) {
fwrite($wwfffilehandle, $row[0] . PHP_EOL);
$nCount++;
}
}
fclose($handle);
fclose($wwfffilehandle);
if ($nCount > 0) {
echo "DONE: " . number_format($nCount) . " WWFF's saved";
} else {
echo "FAILED: Empty file";
}
}
public function update_pota() {
$csvfile = 'https://pota.app/all_parks.csv';
// Text file used for autocomplete in QSO entry
$potafile = './assets/json/pota.txt';
// Persist the full upstream CSV alongside for richer awards features
$potaCsvLocal = './assets/json/pota_parks.csv';
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $csvfile);
curl_setopt($ch, CURLOPT_HEADER, false);
curl_setopt($ch, CURLOPT_USERAGENT, 'Cloudlog Updater');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$csv = curl_exec($ch);
curl_close($ch);
if ($csv === FALSE) {
echo "Something went wrong with fetching the POTA file";
return;
}
// Try to also save a verbatim copy of the CSV for later lookups (park name/coords)
$savedCsvOk = @file_put_contents($potaCsvLocal, $csv) !== FALSE;
$potafilehandle = fopen($potafile, 'w');
if ($potafilehandle === FALSE) {
echo"FAILED: Could not write to pota.txt file";
return;
}
$data = str_getcsv($csv,"\n");
$nCount = 0;
foreach ($data as $idx => $row) {
if ($idx == 0) continue; // Skip line we are not interested in
$row = str_getcsv($row, ',');
if ($row[0]) {
fwrite($potafilehandle, $row[0].PHP_EOL);
$nCount++;
}
}
fclose($potafilehandle);
if ($nCount > 0)
{
echo "DONE: " . number_format($nCount) . " POTA refs saved";
if ($savedCsvOk) {
echo ", CSV cached to assets/json/pota_parks.csv";
} else {
echo ", but failed to cache full CSV";
}
} else {
echo"FAILED: Empty file";
}
}
// UI-friendly wrapper methods with authorization for use from maintenance page
public function update_clublog_scp_ui() {
$this->load->model('user_model');
if(!$this->user_model->authorize(99)) {
echo '<div class="alert alert-danger">You are not authorized to perform this action.</div>';
return;
}
echo '<div class="alert alert-info">';
$this->update_clublog_scp();
echo '</div>';
}
public function lotw_users_ui() {
$this->load->model('user_model');
if(!$this->user_model->authorize(99)) {
echo '<div class="alert alert-danger">You are not authorized to perform this action.</div>';
return;
}
echo '<div class="alert alert-info">';
$this->lotw_users();
echo '</div>';
}
public function update_dok_ui() {
$this->load->model('user_model');
if(!$this->user_model->authorize(99)) {
echo '<div class="alert alert-danger">You are not authorized to perform this action.</div>';
return;
}
echo '<div class="alert alert-info">';
$this->update_dok();
echo '</div>';
}
public function update_sota_ui() {
$this->load->model('user_model');
if(!$this->user_model->authorize(99)) {
echo '<div class="alert alert-danger">You are not authorized to perform this action.</div>';
return;
}
echo '<div class="alert alert-info">';
$this->update_sota();
echo '</div>';
}
public function update_wwff_ui() {
$this->load->model('user_model');
if(!$this->user_model->authorize(99)) {
echo '<div class="alert alert-danger">You are not authorized to perform this action.</div>';
return;
}
echo '<div class="alert alert-info">';
$this->update_wwff();
echo '</div>';
}
public function update_pota_ui() {
$this->load->model('user_model');
if(!$this->user_model->authorize(99)) {
echo '<div class="alert alert-danger">You are not authorized to perform this action.</div>';
return;
}
echo '<div class="alert alert-info">';
$this->update_pota();
echo '</div>';
}
}
?>