Commit 8cd8c8e7 authored by David Anderson's avatar David Anderson

server software: handle 64-bit database IDs

The SETI@home result table is about to run out of 32-bit IDs,
so we need to move to 64-bit result IDs.
This will happen to the workunit table at some point too.

I changed the server C++ code to use the "long" type for all DB IDs
(and to use appropriate conversion codes like %lu).
"long" is 64 bit on 64-bit machines.
For uniformity I did this for all tables,
even ones (like app) that will never get big.

I chose NOT to change the DB schema for now.
The new code will work with 32-bit ID fields in the DB.
As projects approach the 32-bit limit on a table they can change
its ID field, and fields that reference this table, to BIGINT.
This is likely to happen only on the result and workunit tables.
I put functions in html/ops/db_update.php
to change the IDs of these tables.
parent 94e31621
This diff is collapsed.
......@@ -35,12 +35,12 @@
extern DB_CONN boinc_db;
struct TRANSITIONER_ITEM {
int id; // WARNING: this is the WU ID
DB_ID_TYPE id; // WARNING: this is the WU ID
char name[256];
int appid;
DB_ID_TYPE appid;
int min_quorum;
bool need_validate;
int canonical_resultid;
DB_ID_TYPE canonical_resultid;
int transition_time;
int delay_bound;
int error_mask;
......@@ -53,10 +53,10 @@ struct TRANSITIONER_ITEM {
int priority;
int hr_class;
int batch;
int app_version_id;
DB_ID_TYPE app_version_id;
int transitioner_flags;
int size_class;
int res_id; // This is the RESULT ID
DB_ID_TYPE res_id; // This is the RESULT ID
char res_name[256];
int res_report_deadline;
int res_server_state;
......@@ -64,9 +64,9 @@ struct TRANSITIONER_ITEM {
int res_validate_state;
int res_file_delete_state;
int res_sent_time;
int res_hostid;
DB_ID_TYPE res_hostid;
int res_received_time;
int res_app_version_id;
DB_ID_TYPE res_app_version_id;
int res_exit_status;
void clear();
......@@ -88,7 +88,7 @@ struct DB_USER_SUBMIT : public DB_BASE, public USER_SUBMIT {
};
struct STATE_COUNTS {
int appid;
DB_ID_TYPE appid;
int last_update_time;
int result_server_state_2;
int result_server_state_4;
......@@ -262,12 +262,12 @@ public:
int nitems_this_query;
int enumerate(
int appid,
DB_ID_TYPE appid,
int nresult_limit,
int wu_id_modulus,
int wu_id_remainder,
int wu_id_min,
int wu_id_max,
DB_ID_TYPE wu_id_min,
DB_ID_TYPE wu_id_max,
std::vector<VALIDATOR_ITEM>& items
);
int update_result(RESULT&);
......@@ -278,7 +278,7 @@ public:
// used by the feeder and scheduler for outgoing work
//
struct WORK_ITEM {
int res_id;
DB_ID_TYPE res_id;
int res_priority;
int res_server_state;
double res_report_deadline;
......@@ -287,7 +287,7 @@ struct WORK_ITEM {
};
class DB_WORK_ITEM : public WORK_ITEM, public DB_BASE_SPECIAL {
int start_id;
DB_ID_TYPE start_id;
// when enumerate_all is used, keeps track of which ID to start from
public:
DB_WORK_ITEM(DB_CONN* p=0);
......@@ -322,7 +322,7 @@ struct IN_PROGRESS_RESULT {
class DB_IN_PROGRESS_RESULT : public IN_PROGRESS_RESULT, public DB_BASE_SPECIAL {
public:
DB_IN_PROGRESS_RESULT(DB_CONN* p=0);
int enumerate(int hostid, const char* result_names);
int enumerate(DB_ID_TYPE hostid, const char* result_names);
};
// Used by the scheduler to handle results reported by clients
......@@ -331,17 +331,17 @@ public:
struct SCHED_RESULT_ITEM {
char queried_name[256]; // name as reported by client
int id;
DB_ID_TYPE id;
char name[256];
int workunitid;
int appid;
DB_ID_TYPE workunitid;
DB_ID_TYPE appid;
int server_state;
int client_state;
int validate_state;
int outcome;
int hostid;
int userid;
int teamid;
DB_ID_TYPE hostid;
DB_ID_TYPE userid;
DB_ID_TYPE teamid;
int sent_time;
int received_time;
double cpu_time;
......@@ -351,7 +351,7 @@ struct SCHED_RESULT_ITEM {
int exit_status;
int file_delete_state;
double elapsed_time;
int app_version_id;
DB_ID_TYPE app_version_id;
double peak_working_set_size;
double peak_swap_size;
double peak_disk_usage;
......@@ -379,7 +379,7 @@ public:
};
struct FILE_ITEM {
int id;
DB_ID_TYPE id;
char name[254];
char md5sum[34];
double size;
......@@ -397,7 +397,7 @@ public:
};
struct FILESET_ITEM {
int id;
DB_ID_TYPE id;
char name[254];
void clear();
......@@ -416,8 +416,8 @@ public:
};
struct FILESET_FILE_ITEM {
int fileset_id;
int file_id;
DB_ID_TYPE fileset_id;
DB_ID_TYPE file_id;
void clear();
};
......@@ -431,8 +431,8 @@ public:
};
struct SCHED_TRIGGER_ITEM {
int id;
int fileset_id;
DB_ID_TYPE id;
DB_ID_TYPE fileset_id;
bool need_work;
bool work_available;
bool no_work_available;
......
......@@ -38,11 +38,13 @@
struct BEST_APP_VERSION;
typedef long DB_ID_TYPE;
// A compilation target, i.e. a architecture/OS combination.
// Client will be sent applications only for platforms they support.
//
struct PLATFORM {
int id;
DB_ID_TYPE id;
int create_time;
char name[256]; // i.e. "sparc-sun-solaris"
char user_friendly_name[256]; // i.e. "SPARC Solaris 2.8"
......@@ -58,7 +60,7 @@ struct PLATFORM {
// An application.
//
struct APP {
int id;
DB_ID_TYPE id;
int create_time;
char name[256]; // application name, preferably short
int min_version; // don't use app versions before this
......@@ -96,11 +98,11 @@ struct APP {
// A version of an application.
//
struct APP_VERSION {
int id;
DB_ID_TYPE id;
int create_time;
int appid;
DB_ID_TYPE appid;
int version_num;
int platformid;
DB_ID_TYPE platformid;
char xml_doc[APP_VERSION_XML_BLOB_SIZE];
// describes app files. format:
// <file_info>...</file_info>
......@@ -153,7 +155,7 @@ struct APP_VERSION {
};
struct USER {
int id;
DB_ID_TYPE id;
int create_time;
char email_addr[256];
char name[256];
......@@ -180,7 +182,7 @@ struct USER {
// </venue>
// ...
// </project_preferences>
int teamid; // team ID if any
DB_ID_TYPE teamid; // team ID if any
char venue[256]; // home/work/school (default)
char url[256]; // user's web page if any
bool send_email;
......@@ -222,9 +224,9 @@ struct USER {
// a team has > 0 members
struct TEAM {
int id;
DB_ID_TYPE id;
int create_time;
int userid; // User ID of team founder
DB_ID_TYPE userid; // User ID of team founder
char name[256];
char name_lc[256]; // Team name in lowercase (used for searching)
char url[256];
......@@ -247,9 +249,9 @@ struct TEAM {
};
struct HOST {
int id;
DB_ID_TYPE id;
int create_time;
int userid; // ID of user running this host
DB_ID_TYPE userid; // ID of user running this host
// If the host is "zombied" during merging of duplicate hosts,
// this field is set to zero and rpc_seqno is used to
// store the ID of the new host (kludge, but what the heck)
......@@ -407,9 +409,9 @@ struct HOST {
// transition, but don't create results; used for targeted jobs
struct WORKUNIT {
int id;
DB_ID_TYPE id;
int create_time;
int appid; // associated app
DB_ID_TYPE appid; // associated app
char name[256];
char xml_doc[BLOB_SIZE];
int batch;
......@@ -434,7 +436,7 @@ struct WORKUNIT {
// 2) abort task if it uses more than this disk
bool need_validate; // this WU has at least 1 successful result in
// validate state = INIT
int canonical_resultid; // ID of canonical result, or zero
DB_ID_TYPE canonical_resultid; // ID of canonical result, or zero
double canonical_credit; // credit that all correct results get
// TODO: deprecate and remove code
int transition_time; // when should transition_handler
......@@ -464,8 +466,8 @@ struct WORKUNIT {
char mod_time[16];
double rsc_bandwidth_bound;
// send only to hosts with at least this much download bandwidth
int fileset_id;
int app_version_id;
DB_ID_TYPE fileset_id;
DB_ID_TYPE app_version_id;
// if app uses homogeneous_app_version,
// which version this job is committed to (0 if none)
int transitioner_flags;
......@@ -481,8 +483,8 @@ struct WORKUNIT {
};
struct CREDITED_JOB {
int userid;
double workunitid;
DB_ID_TYPE userid;
DB_ID_TYPE workunitid;
// the following not used in the DB
void clear();
......@@ -555,9 +557,9 @@ struct CREDITED_JOB {
#define ANON_PLATFORM_INTEL -5
struct RESULT {
int id;
DB_ID_TYPE id;
int create_time;
int workunitid;
DB_ID_TYPE workunitid;
int server_state; // see above
int outcome; // see above; defined if server state OVER
int client_state; // phase that client contacted us in.
......@@ -565,8 +567,8 @@ struct RESULT {
// error details are in stderr_out.
// The values for this field are defined
// in lib/result_state.h
int hostid; // host processing this result
int userid; // user processing this result
DB_ID_TYPE hostid; // host processing this result
DB_ID_TYPE userid; // user processing this result
int report_deadline; // deadline for receiving result
int sent_time; // when result was sent to host
int received_time; // when result was received from host
......@@ -584,16 +586,16 @@ struct RESULT {
int random; // determines send order
int app_version_num; // version# of app
// DEPRECATED - THIS DOESN'T DETERMINE VERSION ANY MORE
int appid; // copy of WU's appid
DB_ID_TYPE appid; // copy of WU's appid
int exit_status; // application exit status, if any
int teamid;
DB_ID_TYPE teamid;
int priority;
char mod_time[16];
double elapsed_time;
// AKA runtime; returned by 6.10+ clients
double flops_estimate;
// misnomer: actually the peak device FLOPS, returned by app_plan().
int app_version_id;
DB_ID_TYPE app_version_id;
// ID of app version used to compute this
// 0 if unknown (relic of old scheduler)
// -1 anon platform, unknown resource type (relic)
......@@ -614,8 +616,8 @@ struct RESULT {
};
struct BATCH {
int id;
int user_id;
DB_ID_TYPE id;
DB_ID_TYPE user_id;
// submitter
int create_time;
double logical_start_time;
......@@ -640,7 +642,7 @@ struct BATCH {
// the sum of credits of all results
char name[256];
// user-assigned name; need not be unique
int app_id;
DB_ID_TYPE app_id;
int project_state;
// project-assigned
char description[256];
......@@ -653,7 +655,7 @@ struct BATCH {
// info for users who can submit jobs
//
struct USER_SUBMIT {
int user_id;
DB_ID_TYPE user_id;
double quota;
double logical_start_time;
bool submit_all;
......@@ -662,9 +664,9 @@ struct USER_SUBMIT {
};
struct MSG_FROM_HOST {
int id;
DB_ID_TYPE id;
int create_time;
int hostid;
DB_ID_TYPE hostid;
char variety[256]; // project-defined; what kind of msg
bool handled; // message handler has processed this
char xml[MSG_FROM_HOST_BLOB_SIZE];
......@@ -672,9 +674,9 @@ struct MSG_FROM_HOST {
};
struct MSG_TO_HOST {
int id;
DB_ID_TYPE id;
int create_time;
int hostid;
DB_ID_TYPE hostid;
char variety[256]; // project-defined; what kind of msg
bool handled; // scheduler has sent this
char xml[MSG_TO_HOST_BLOB_SIZE]; // text to include in sched reply
......@@ -682,20 +684,20 @@ struct MSG_TO_HOST {
};
struct ASSIGNMENT {
int id;
DB_ID_TYPE id;
int create_time;
int target_id; // ID of target host, user, or team
DB_ID_TYPE target_id; // ID of target host, user, or team
int target_type; // none/host/user/team
int multi; // 0 = single host, 1 = all hosts in set
int workunitid;
int _resultid; // if not multi, the result ID
DB_ID_TYPE workunitid;
DB_ID_TYPE _resultid; // if not multi, the result ID
// deprecated
void clear();
};
struct HOST_APP_VERSION {
int host_id;
int app_version_id;
DB_ID_TYPE host_id;
DB_ID_TYPE app_version_id;
// or for anon platform:
// 1000000*appid + 2 (CPU)
// 1000000*appid + 3 (NVIDIA)
......@@ -732,7 +734,7 @@ struct HOST_APP_VERSION {
};
struct VDA_FILE {
int id;
DB_ID_TYPE id;
double create_time;
char dir[256];
char file_name[256];
......@@ -747,8 +749,8 @@ struct VDA_FILE {
struct VDA_CHUNK_HOST {
double create_time;
int vda_file_id;
int host_id;
DB_ID_TYPE vda_file_id;
DB_ID_TYPE host_id;
char physical_file_name[256]; // e.g. vda_467_0_file.ext
bool present_on_host;
bool transfer_in_progress;
......@@ -770,7 +772,7 @@ struct VDA_CHUNK_HOST {
};
struct BADGE {
int id;
DB_ID_TYPE id;
double create_time;
int type;
char name[256];
......@@ -784,24 +786,24 @@ struct BADGE {
};
struct BADGE_USER {
int badge_id;
int user_id;
DB_ID_TYPE badge_id;
DB_ID_TYPE user_id;
double create_time;
double reassign_time;
void clear();
};
struct BADGE_TEAM {
int badge_id;
int team_id;
DB_ID_TYPE badge_id;
DB_ID_TYPE team_id;
double create_time;
double reassign_time;
void clear();
};
struct CREDIT_USER {
int userid;
int appid;
DB_ID_TYPE userid;
DB_ID_TYPE appid;
// need not be an app ID
int njobs;
double total;
......@@ -812,8 +814,8 @@ struct CREDIT_USER {
};
struct CREDIT_TEAM {
int teamid;
int appid;
DB_ID_TYPE teamid;
DB_ID_TYPE appid;
int njobs;
double total;
double expavg;
......
......@@ -215,13 +215,13 @@ int DB_BASE::affected_rows() {
//////////// FUNCTIONS FOR TABLES THAT HAVE AN ID FIELD ///////
int DB_BASE::lookup_id(int id) {
int DB_BASE::lookup_id(DB_ID_TYPE id) {
char query[MAX_QUERY_LEN];
int retval;
MYSQL_ROW row;
MYSQL_RES* rp;
sprintf(query, "select * from %s where id=%u", table_name, id);
sprintf(query, "select * from %s where id=%lu", table_name, id);
retval = db->do_query(query);
if (retval) return retval;
......@@ -321,10 +321,10 @@ int DB_BASE::get_field_str(const char* field, char* buf, int buflen) {
return 0;
}
int DB_BASE::max_id(int& n, const char* clause) {
int DB_BASE::max_id(DB_ID_TYPE& n, const char* clause) {
char query[MAX_QUERY_LEN];
sprintf(query, "select max(id) from %s %s", table_name, clause);
return get_integer(query, n);
return get_long(query, n);
}
/////////////// FUNCTIONS THAT DON'T REQUIRE AN ID FIELD ///////////////
......@@ -407,7 +407,7 @@ int DB_BASE::end_enumerate() {
return 0;
}
int DB_BASE::get_integer(const char* query, int& n) {
int DB_BASE::get_long(const char* query, long& n) {
int retval;
MYSQL_ROW row;
MYSQL_RES* resp;
......@@ -420,7 +420,7 @@ int DB_BASE::get_integer(const char* query, int& n) {
if (!row || !row[0]) {
retval = ERR_DB_NOT_FOUND;
} else {
n = atoi(row[0]);
n = atol(row[0]);
}
mysql_free_result(resp);
return retval;
......@@ -445,12 +445,12 @@ int DB_CONN::get_double(const char* query, double& x) {
return retval;
}
int DB_BASE::count(int& n, const char* clause) {
int DB_BASE::count(long& n, const char* clause) {
char query[MAX_QUERY_LEN];
sprintf(query, "select count(*) from %s %s", table_name, clause);
return get_integer(query, n);
return get_long(query, n);
}
int DB_BASE::sum(double& x, const char* field, const char* clause) {
......
......@@ -33,6 +33,11 @@ inline int safe_atoi(const char* s) {
return atoi(s);
}
inline long safe_atol(const char* s) {
if (!s) return 0;
return atol(s);
}
inline double safe_atof(const char* s) {
if (!s) return 0;
return atof(s);
......@@ -64,6 +69,8 @@ enum ISOLATION_LEVEL {
SERIALIZABLE
};
typedef long DB_ID_TYPE;
// represents a connection to a database
//
class DB_CONN {
......@@ -102,14 +109,14 @@ public:
int delete_from_db_multi(const char* where_clause);
int get_field_ints(const char*, int, int*);
int get_field_str(const char*, char*, int);
int lookup_id(int id);
int lookup_id(DB_ID_TYPE id);
int lookup(const char*);
int enumerate(const char* clause="", bool use_use_result=false);
int end_enumerate();
int count(int&, const char* clause="");
int max_id(int&, const char* clause="");
int count(long&, const char* clause="");
int max_id(DB_ID_TYPE&, const char* clause="");
int sum(double&, const char* field, const char* clause="");
int get_integer(const char* query, int&);
int get_long(const char* query, long&);
int affected_rows();
DB_CONN* db;
......
......@@ -20,10 +20,10 @@ $sensing = array(
array(
array(
"Quake Catcher Network",
"http://qcn.stanford.edu/sensor/",
tra("Stanford University"),
"http://qcn.caltech.edu/sensor",
tra("Cal Tech"),
tra("Seismology"),
tra("Quake-Catcher Network is developing the world's largest seismic network using sensors attached to computers and smartphones."),
tra("Quake-Catcher Network uses sensors attached to computers and smartphones to detect seismic waves."),
"",
"",
"Help detect earthquakes"
......
......@@ -989,6 +989,32 @@ function update_4_15_2015() {
");
}
// functions to change select ID types to 64-bit
//
function result_big_ids() {
do_query("alter table result
change column id id bigint not null auto_increment
");
do_query("alter table workunit
change column canonical_resultid canonical_resultid bigint not null
");
do_query("alter table assignment
change column resultid resultid bigint not null
");
}
function workunit_big_ids() {
do_query("alter table workunit
change column id id bigint not null auto_increment
");
do_query("alter table result
change column workunitid workunitid bigint not null
");
do_query("alter table assignment
change column workunitid workunitid bigint not null
");
}
// Updates are done automatically if you use "upgrade".
//
// If you need to do updates manually,
......
......@@ -661,6 +661,41 @@ bool XML_PARSER::parse_int(const char* start_tag, int& i) {
return true;
}
// Same, for long
//
bool XML_PARSER::parse_long(const char* start_tag, long& i) {
char buf[256], *end;
bool eof;
char end_tag[TAG_BUF_LEN], tag[TAG_BUF_LEN];
if (strcmp(parsed_tag, start_tag)) return false;
end_tag[0] = '/';
strcpy(end_tag+1, start_tag);
eof = get(buf, sizeof(buf), is_tag);
if (eof) return false;
if (is_tag) {
if (!strcmp(buf, end_tag)) {
i = 0; // treat <foo></foo> as <foo>0</foo>
return true;
} else {
return false;
}
}
errno = 0;
long val = strtol(buf, &end, 0);
if (errno) return false;
if (end != buf+strlen(buf)) return false;
eof = get(tag, sizeof(tag), is_tag);
if (eof) return false;
<