vboxwrapper.cpp 44.1 KB
Newer Older
Rom Walton's avatar
Rom Walton committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
// This file is part of BOINC.
// http://boinc.berkeley.edu
// Copyright (C) 2010-2012 University of California
//
// BOINC is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License
// as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// BOINC is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with BOINC.  If not, see <http://www.gnu.org/licenses/>.

18
// BOINC VirtualBox wrapper; lets you run apps in VMs
Rom Walton's avatar
Rom Walton committed
19
// see: http://boinc.berkeley.edu/trac/wiki/VboxApps
20
21
22
//
// usage: vboxwrapper [options]
//
23
// --trickle X      send a trickle-up message reporting elapsed time every X sec
Rom Walton's avatar
Rom Walton committed
24
25
//                  (use this for credit granting if your app does its
//                  own job management, like CernVM).
26
// --nthreads N     create a VM with N threads.
27
// --vmimage N      Use "vm_image_N" as the VM image.
28
29
30
31
32
//                  This lets you create an app version with several images,
//                  and the app_plan function can decide which one to use
//                  for the particular host.
// --register_only  Register the VM but don't run it.
//                  Useful for debugging; see the wiki page
Rom Walton's avatar
Rom Walton committed
33
34
35
36
//
// Handles:
// - suspend/resume/quit/abort
// - reporting CPU time
37
38
39
// - loss of heartbeat from client
// - checkpoint (using snapshots)
// - a bunch of other stuff; see the wiki page
Rom Walton's avatar
Rom Walton committed
40
41
//
// Contributors:
42
43
// Rom Walton
// David Anderson
Rom Walton's avatar
Rom Walton committed
44
45
46
47
48
49
50
51
52
53
54
55
56
// Andrew J. Younge (ajy4490 AT umiacs DOT umd DOT edu)
// Jie Wu <jiewu AT cern DOT ch>
// Daniel Lombraña González <teleyinex AT gmail DOT com>

#ifdef _WIN32
#include "boinc_win.h"
#include "win_util.h"
#else
#include <vector>
#include <sys/wait.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>
57
#include <cmath>
Rom Walton's avatar
Rom Walton committed
58
#include <string>
59
#include <sstream>
Rom Walton's avatar
Rom Walton committed
60
61
62
#include <unistd.h>
#endif

63
#include "version.h"
Rom Walton's avatar
Rom Walton committed
64
#include "boinc_api.h"
65
#include "graphics2.h"
Rom Walton's avatar
Rom Walton committed
66
67
#include "diagnostics.h"
#include "filesys.h"
68
#include "md5_file.h"
Rom Walton's avatar
Rom Walton committed
69
70
71
72
73
74
#include "parse.h"
#include "str_util.h"
#include "str_replace.h"
#include "util.h"
#include "error_numbers.h"
#include "procinfo.h"
75
#include "floppyio.h"
76
77
#include "vboxlogging.h"
#include "vboxcheckpoint.h"
Rom Walton's avatar
Rom Walton committed
78
#include "vboxwrapper.h"
79
#include "vbox_common.h"
80
#ifdef _WIN32
81
82
#include "vbox_mscom42.h"
#include "vbox_mscom43.h"
83
#include "vbox_mscom50.h"
84
#endif
85
86
#include "vbox_vboxmanage.h"

87

Rom Walton's avatar
Rom Walton committed
88
89
90
using std::vector;
using std::string;

91

Rom Walton's avatar
Rom Walton committed
92
void read_fraction_done(double& frac_done, VBOX_VM& vm) {
93
    char path[MAXPATHLEN];
Rom Walton's avatar
Rom Walton committed
94
95
96
    char buf[256];
    double temp, frac = 0;

97
98
    sprintf(path, "shared/%s", vm.fraction_done_filename.c_str());
    FILE* f = fopen(path, "r");
Rom Walton's avatar
Rom Walton committed
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    if (!f) return;

    // read the last line of the file
    //
    fseek(f, -32, SEEK_END);
    while (!feof(f)) {
        char* p = fgets(buf, 256, f);
        if (p == NULL) break;
        int n = sscanf(buf, "%lf", &temp);
        if (n == 1) frac = temp;
    }
    fclose(f);

    if (frac < 0) {
        frac = 0;
    }
    if (frac > 1) {
        frac = 1;
    }

    frac_done = frac;
}

122
123
124
bool completion_file_exists(VBOX_VM& vm) {
    char path[MAXPATHLEN];
    sprintf(path, "shared/%s", vm.completion_trigger_file.c_str());
125
    if (vm.completion_trigger_file.size() && boinc_file_exists(path)) return true;
126
127
128
    return false;
}

129
void read_completion_file_info(unsigned long& exit_code, bool& is_notice, string& message, VBOX_VM& vm) {
130
131
132
133
134
135
136
137
138
139
    char path[MAXPATHLEN];
    char buf[1024];

    exit_code = 0;
    message = "";

    sprintf(path, "shared/%s", vm.completion_trigger_file.c_str());
    FILE* f = fopen(path, "r");
    if (f) {
        if (fgets(buf, 1024, f) != NULL) {
Rom Walton's avatar
Rom Walton committed
140
            exit_code = atoi(buf) != 0;
141
        }
142
        if (fgets(buf, 1024, f) != NULL) {
Rom Walton's avatar
Rom Walton committed
143
            is_notice = atoi(buf) != 0;
144
        }
145
146
147
148
149
150
151
        while (fgets(buf, 1024, f) != NULL) {
            message += buf;
        }
        fclose(f);
    }
}

152
153
154
bool temporary_exit_file_exists(VBOX_VM& vm) {
    char path[MAXPATHLEN];
    sprintf(path, "shared/%s", vm.temporary_exit_trigger_file.c_str());
155
    if (vm.temporary_exit_trigger_file.size() && boinc_file_exists(path)) return true;
156
157
158
    return false;
}

159
160
161
162
163
164
165
166
167
168
169
170
171
172
void read_temporary_exit_file_info(int& temp_delay, bool& is_notice, string& message, VBOX_VM& vm) {
    char path[MAXPATHLEN];
    char buf[1024];

    temp_delay = 0;
    message = "";

    sprintf(path, "shared/%s", vm.temporary_exit_trigger_file.c_str());
    FILE* f = fopen(path, "r");
    if (f) {
        if (fgets(buf, 1024, f) != NULL) {
            temp_delay = atoi(buf);
        }
        if (fgets(buf, 1024, f) != NULL) {
Rom Walton's avatar
Rom Walton committed
173
            is_notice = atoi(buf) != 0;
174
175
176
177
178
179
180
181
        }
        while (fgets(buf, 1024, f) != NULL) {
            message += buf;
        }
        fclose(f);
    }
}

182
183
184
185
186
187
void delete_temporary_exit_trigger_file(VBOX_VM& vm) {
    char path[MAXPATHLEN];
    sprintf(path, "shared/%s", vm.temporary_exit_trigger_file.c_str());
    boinc_delete_file(path);
}

Rom Walton's avatar
Rom Walton committed
188
189
190
// set CPU and network throttling if needed
//
void set_throttles(APP_INIT_DATA& aid, VBOX_VM& vm) {
191
    double x = 0, y = 0;
192

193
    // VirtualBox freaks out if the CPU Usage value is too low to actually
194
195
196
    // do any processing.  It probably wouldn't be so bad if the RDP interface
    // didn't also get hosed by it.
    //
197
198
199
200
    x = aid.global_prefs.cpu_usage_limit;
    // 0 means "no limit"
    //
    if (x == 0.0) x = 100;
201
    // For now set the minimum CPU Usage value to 1.
202
    //
203
204
    if (x < 1) x = 1;
    vm.set_cpu_usage((int)x);
Rom Walton's avatar
Rom Walton committed
205
206
207
208

    // vbox doesn't distinguish up and down bandwidth; use the min of the prefs
    //
    x = aid.global_prefs.max_bytes_sec_up;
209
    y = aid.global_prefs.max_bytes_sec_down;
Rom Walton's avatar
Rom Walton committed
210
    if (y) {
211
        if (!x || y < x) {
Rom Walton's avatar
Rom Walton committed
212
213
214
215
            x = y;
        }
    }
    if (x) {
216
        vm.set_network_usage((int)(x/1024));
Rom Walton's avatar
Rom Walton committed
217
    }
218

Rom Walton's avatar
Rom Walton committed
219
220
221
}

// If the Floppy device has been specified, initialize its state so that
222
223
224
225
// it contains the contents of the init_data.xml file.
// In theory this would allow network enabled VMs to know about
// proxy server configurations either specified by the volunteer
// or automatically detected by the client.
Rom Walton's avatar
Rom Walton committed
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
//
// CERN decided they only needed a small subset of the data and changed the
// data format to 'name=value\n' pairs.  So if we are running under their
// environment set things up accordingly.
//
void set_floppy_image(APP_INIT_DATA& aid, VBOX_VM& vm) {
    int retval;
    char buf[256];
    std::string scratch;

    if (vm.enable_floppyio) {
        scratch = "";
        if (!vm.enable_cern_dataformat) {
            retval = read_file_string(INIT_DATA_FILE, scratch);
            if (retval) {
241
                vboxlog_msg("WARNING: Cannot write init_data.xml to floppy abstration device");
Rom Walton's avatar
Rom Walton committed
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
            }
        } else {
            // Per: https://github.com/stig/json-framework/issues/48
            //
            // Use %.17g to represent doubles
            //
            scratch  = "BOINC_USERNAME=" + string(aid.user_name) + "\n";
            scratch += "BOINC_AUTHENTICATOR=" + string(aid.authenticator) + "\n";

            sprintf(buf, "%d", aid.userid);
            scratch += "BOINC_USERID=" + string(buf) + "\n";

            sprintf(buf, "%d", aid.hostid);
            scratch += "BOINC_HOSTID=" + string(buf) + "\n";

            sprintf(buf, "%.17g", aid.user_total_credit);
            scratch += "BOINC_USER_TOTAL_CREDIT=" + string(buf) + "\n";

            sprintf(buf, "%.17g", aid.host_total_credit);
            scratch += "BOINC_HOST_TOTAL_CREDIT=" + string(buf) + "\n";
        }
        vm.write_floppy(scratch);
    }
}

267
// if there's a port for web graphics, tell the client about it
Rom Walton's avatar
Rom Walton committed
268
//
269
void report_web_graphics_url(VBOX_VM& vm) {
270
    char buf[256];
271
272
273
274
    if (vm.pf_host_port && !boinc_file_exists("graphics_app")) {
        sprintf(buf, "http://localhost:%d", vm.pf_host_port);
        vboxlog_msg("Detected: Web Application Enabled (%s)", buf);
        boinc_web_graphics_url(buf);
Rom Walton's avatar
Rom Walton committed
275
276
277
278
279
    }
}

// set remote desktop information if needed
//
280
void report_remote_desktop_info(VBOX_VM& vm) {
Rom Walton's avatar
Rom Walton committed
281
282
283
    char buf[256];
    if (vm.rd_host_port) {
        sprintf(buf, "localhost:%d", vm.rd_host_port);
284
        vboxlog_msg("Detected: Remote Desktop Enabled (%s)", buf);
Rom Walton's avatar
Rom Walton committed
285
286
287
288
        boinc_remote_desktop_addr(buf);
    }
}

289
290
// check for trickle trigger files, and send trickles if find them.
//
291
void check_trickle_triggers(VBOX_VM& vm) {
292
293
294
    int retval;
    char filename[256], path[MAXPATHLEN];
    std::string text;
295
296
    for (unsigned int i=0; i<vm.trickle_trigger_files.size(); i++) {
        strcpy(filename, vm.trickle_trigger_files[i].c_str());
297
298
        sprintf(path, "shared/%s", filename);
        if (!boinc_file_exists(path)) continue;
299
300
        vboxlog_msg("Reporting a trickle. (%s)", filename);
        retval = read_file_string(path, text);
301
        if (retval) {
302
            vboxlog_msg("ERROR: can't read trickle trigger file %s", filename);
303
304
305
306
307
        } else {
            retval = boinc_send_trickle_up(
                filename, const_cast<char*>(text.c_str())
            );
            if (retval) {
308
                vboxlog_msg("boinc_send_trickle_up() failed: %s (%d)", boincerror(retval), retval);
309
            }
310
311
312
313
314
        }
        boinc_delete_file(path);
    }
}

315
316
// check for intermediate upload files, and send them if found.
//
317
void check_intermediate_uploads(VBOX_VM& vm) {
318
    int retval;
319
    char filename[256], path[MAXPATHLEN];
320
321
    for (unsigned int i=0; i<vm.intermediate_upload_files.size(); i++) {
        strcpy(filename, vm.intermediate_upload_files[i].file.c_str());
322
323
        sprintf(path, "shared/%s", filename);
        if (!boinc_file_exists(path)) continue;
324
        if (!vm.intermediate_upload_files[i].reported && !vm.intermediate_upload_files[i].ignore) {
325
            vboxlog_msg("Reporting an intermediate file. (%s)", vm.intermediate_upload_files[i].file.c_str());
326
            retval = boinc_upload_file(vm.intermediate_upload_files[i].file);
327
            if (retval) {
328
                vboxlog_msg("boinc_upload_file() failed: %s", boincerror(retval));
329
                vm.intermediate_upload_files[i].ignore = true;
330
            } else {
331
                vm.intermediate_upload_files[i].reported = true;
332
            }
333
334
        } else if (vm.intermediate_upload_files[i].reported && !vm.intermediate_upload_files[i].ignore) {
            retval = boinc_upload_status(vm.intermediate_upload_files[i].file);
335
            if (!retval) {
336
                vboxlog_msg("Intermediate file uploaded. (%s)", vm.intermediate_upload_files[i].file.c_str());
337
                vm.intermediate_upload_files[i].ignore = true;
338
339
340
341
342
            }
        }
    }
}

343
344
// see if it's time to send trickle-up reporting elapsed time
//
345
void check_trickle_period(double& elapsed_time, double& trickle_period) {
346
347
348
349
350
351
352
    char buf[256];
    static double last_trickle_report_time = 0;

    if ((elapsed_time - last_trickle_report_time) < trickle_period) {
        return;
    }
    last_trickle_report_time = elapsed_time;
353
    vboxlog_msg("Status Report: Trickle-Up Event.");
354
355
356
357
358
359
360
    sprintf(buf,
        "<cpu_time>%f</cpu_time>", last_trickle_report_time
    );
    int retval = boinc_send_trickle_up(
        const_cast<char*>("cpu_time"), buf
    );
    if (retval) {
361
        vboxlog_msg("Sending Trickle-Up Event failed (%d).", retval);
362
363
364
    }
}

Rom Walton's avatar
Rom Walton committed
365
int main(int argc, char** argv) {
366
    int retval = 0;
367
    int loop_iteration = 0;
Rom Walton's avatar
Rom Walton committed
368
369
    BOINC_OPTIONS boinc_options;
    APP_INIT_DATA aid;
370
    VBOX_VM* pVM = NULL;
371
    VBOX_CHECKPOINT checkpoint;
372
    double desired_checkpoint_interval = 0;
373
    double random_checkpoint_factor = 0;
374
    double elapsed_time = 0;
Rom Walton's avatar
Rom Walton committed
375
    double fraction_done = 0;
376
    double trickle_period = 0;
377
    double current_cpu_time = 0;
378
    double starting_cpu_time = 0;
379
380
    double last_checkpoint_cpu_time = 0;
    double last_checkpoint_elapsed_time = 0;
Rom Walton's avatar
Rom Walton committed
381
    double last_status_report_time = 0;
382
    double stopwatch_starttime = 0;
Rom Walton's avatar
Rom Walton committed
383
    double stopwatch_endtime = 0;
384
    double stopwatch_elapsedtime = 0;
Rom Walton's avatar
Rom Walton committed
385
386
387
388
    double sleep_time = 0;
    double bytes_sent = 0;
    double bytes_received = 0;
    double ncpus = 0;
389
    double memory_size_mb = 0;
390
    double timeout = 0.0;
Rom Walton's avatar
Rom Walton committed
391
    bool report_net_usage = false;
392
    double net_usage_timer = 600;
393
	int vm_image = 0;
Rom Walton's avatar
Rom Walton committed
394
    unsigned long vm_exit_code = 0;
395
    bool is_notice = false;
396
    int temp_delay = 86400;
397
    string message;
398
    string scratch_dir;
Rom Walton's avatar
Rom Walton committed
399
400
    char buf[256];

401
402
403
404
405
406
    // Initialize diagnostics system
    //
    boinc_init_diagnostics(BOINC_DIAG_DEFAULTS);

    // Configure BOINC Runtime System environment
    //
407
408
409
410
411
412
413
414
    memset(&boinc_options, 0, sizeof(boinc_options));
    boinc_options.main_program = true;
    boinc_options.check_heartbeat = true;
    boinc_options.handle_process_control = true;
    boinc_init_options(&boinc_options);

    // Log banner
    //
415
    vboxlog_msg("vboxwrapper (%d.%d.%d): starting", BOINC_MAJOR_VERSION, BOINC_MINOR_VERSION, VBOXWRAPPER_RELEASE);
Rom Walton's avatar
Rom Walton committed
416

417
418
419
420
421
422
423
424
    // Initialize system services
    // 
#ifdef _WIN32
    CoInitialize(NULL);
#ifdef USE_WINSOCK
    WSADATA wsdata;
    retval = WSAStartup( MAKEWORD( 1, 1 ), &wsdata);
    if (retval) {
425
        vboxlog_msg("ERROR: Cannot initialize winsock: %d", retval);
426
427
428
429
430
        boinc_finish(retval);
    }
#endif
#endif

431
432
433
434
    // Prepare environment for detecting system conditions
    //
    boinc_parse_init_data_file();
    boinc_get_init_data(aid);
435
436

#ifdef _WIN32
437
438
439
440
441
442
443
444
445
446
447
448
    // Determine what version of VirtualBox we are using via the registry. Use a
    // namespace specific version of the function because VirtualBox has been known
    // to change the registry location from time to time.
    //
    // NOTE: We cannot use COM to automatically detect which interfaces are installed
    //       on the machine because it will attempt to launch the 'vboxsvc' process
    //       without out environment variable changes and muck everything up.
    //
    string vbox_version;
    int vbox_major = 0, vbox_minor = 0;

    if (BOINC_SUCCESS != vbox42::VBOX_VM::get_version_information(vbox_version)) {
449
450
451
        if (BOINC_SUCCESS != vbox43::VBOX_VM::get_version_information(vbox_version)) {
            vbox50::VBOX_VM::get_version_information(vbox_version);
        }
452
453
454
455
456
457
458
459
460
    }
    if (!vbox_version.empty()) {
        sscanf(vbox_version.c_str(), "%d.%d", &vbox_major, &vbox_minor);
        if ((4 == vbox_major) && (2 == vbox_minor)) {
            pVM = (VBOX_VM*) new vbox42::VBOX_VM();
        }
        if ((4 == vbox_major) && (3 == vbox_minor)) {
            pVM = (VBOX_VM*) new vbox43::VBOX_VM();
        }
461
462
463
        if ((5 == vbox_major) && (0 == vbox_minor)) {
            pVM = (VBOX_VM*) new vbox50::VBOX_VM();
        }
464
465
    }
    if (!pVM) {
466
467
468
469
470
471
        pVM = (VBOX_VM*) new vboxmanage::VBOX_VM();
    }
#else
    pVM = (VBOX_VM*) new vboxmanage::VBOX_VM();
#endif

472
473
    // Parse command line parameters
    //
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
    for (int i=1; i<argc; i++) {
        if (!strcmp(argv[i], "--trickle")) {
            trickle_period = atof(argv[++i]);
        }
        if (!strcmp(argv[i], "--ncpus")) {
            ncpus = atof(argv[++i]);
        }
        if (!strcmp(argv[i], "--memory_size_mb")) {
            memory_size_mb = atof(argv[++i]);
        }
        if (!strcmp(argv[i], "--vmimage")) {
            vm_image = atoi(argv[++i]);
        }
        if (!strcmp(argv[i], "--register_only")) {
            pVM->register_only = true;
        }
    }
491
492
493

    // Choose a random interleave value for checkpoint intervals to stagger disk I/O.
    // 
494
495
496
497
498
499
    struct stat vm_image_stat;
    if (-1 == stat(IMAGE_FILENAME_COMPLETE, &vm_image_stat)) {
        srand((int)time(NULL));
    } else {
        srand((int)(vm_image_stat.st_mtime * time(NULL)));
    }
500
    random_checkpoint_factor = (double)(((int)(drand() * 100000.0)) % 600);
501

502
    vboxlog_msg("Feature: Checkpoint interval offset (%d seconds)", (int)random_checkpoint_factor);
503
504
505
506

    // Display trickle value if specified
    //
    if (trickle_period > 0.0) {
507
        vboxlog_msg("Feature: Enabling trickle-ups (Interval: %f)", trickle_period);
508
509
    }

510
511
512
513
    // Check for architecture incompatibilities
    // 
#if defined(_WIN32) && defined(_M_IX86)
    if (strstr(aid.host_info.os_version, "x64")) {
514
        vboxlog_msg("64-bit version of BOINC is required, please upgrade. Rescheduling execution for a later date.");
515
516
517
518
        boinc_temporary_exit(86400, "Architecture incompatibility detected.");
    }
#endif

519
520
    // Initialize VM Hypervisor
    //
521
    retval = pVM->initialize();
522
    if (retval) {
523
        vboxlog_msg("Could not detect VM Hypervisor. Rescheduling execution for a later date.");
524
        boinc_temporary_exit(86400, "Detection of VM Hypervisor failed.");
525
526
    }

527
528
    // Record what version of VirtualBox was used.
    // 
529
    if (!pVM->virtualbox_version.empty()) {
530
        vboxlog_msg("Detected: %s", pVM->virtualbox_version.c_str());
531
532
    }

533
534
535
    // Record if anonymous platform was used.
    // 
    if (boinc_file_exists((std::string(aid.project_dir) + std::string("/app_info.xml")).c_str())) {
536
        vboxlog_msg("Detected: Anonymous Platform Enabled");
537
538
539
540
541
    }

    // Record if the sandboxed configuration is going to be used.
    //
    if (aid.using_sandbox) {
542
        vboxlog_msg("Detected: Sandbox Configuration Enabled");
543
544
    }

Rom Walton's avatar
Rom Walton committed
545
    // Record which mode VirtualBox should be started in.
546
    //
547
    if (aid.vbox_window || boinc_is_standalone()) {
548
        vboxlog_msg("Detected: Headless Mode Disabled");
549
        pVM->headless = false;
550
551
552
553
554
    }

    // Check for invalid confgiurations.
    //
    if (aid.using_sandbox && aid.vbox_window) {
555
556
        vboxlog_msg("Invalid configuration detected.");
        vboxlog_msg("NOTE: BOINC cannot be installed as a service and run VirtualBox in headfull mode at the same time.");
Rom Walton's avatar
Rom Walton committed
557
        boinc_temporary_exit(86400, "Incompatible configuration detected.");
558
559
    }

560
    // Check against known incompatible versions of VirtualBox.  
561
562
    // VirtualBox 4.2.6 crashes during snapshot operations
    // and 4.2.18 fails to restore from snapshots properly.
563
    //
564
565
566
    if ((pVM->virtualbox_version.find("4.2.6") != std::string::npos) || 
        (pVM->virtualbox_version.find("4.2.18") != std::string::npos) || 
        (pVM->virtualbox_version.find("4.3.0") != std::string::npos) ) {
567
        vboxlog_msg("Incompatible version of VirtualBox detected. Please upgrade to a later version.");
568
569
570
571
        boinc_temporary_exit(86400,
            "Incompatible version of VirtualBox detected; please upgrade.",
            true
        );
572
573
    }

574
575
576
577
    // Check to see if the system is in a state in which we expect to be able to run
    // VirtualBox successfully.  Sometimes the system is in a wierd state after a
    // reboot and the system needs a little bit of time.
    //
578
    if (!pVM->is_system_ready(message)) {
579
        vboxlog_msg("Could not communicate with VM Hypervisor. Rescheduling execution for a later date.");
580
        boinc_temporary_exit(86400, message.c_str());
581
582
    }

583
584
    // Parse Job File
    //
585
    retval = pVM->parse();
Rom Walton's avatar
Rom Walton committed
586
    if (retval) {
587
        vboxlog_msg("ERROR: Cannot parse job file: %d", retval);
Rom Walton's avatar
Rom Walton committed
588
589
590
        boinc_finish(retval);
    }

591
    // Record what the minimum checkpoint interval is.
592
    //
593
    vboxlog_msg("Detected: Minimum checkpoint interval (%f seconds)", pVM->minimum_checkpoint_interval);
594

Rom Walton's avatar
Rom Walton committed
595
596
    // Validate whatever configuration options we can
    //
597
    if (pVM->enable_shared_directory) {
598
        pVM->get_scratch_directory(scratch_dir);
Rom Walton's avatar
Rom Walton committed
599
600
        if (boinc_file_exists("shared")) {
            if (!is_dir("shared")) {
601
                vboxlog_msg("ERROR: 'shared' exists but is not a directory.");
Rom Walton's avatar
Rom Walton committed
602
603
604
605
            }
        } else {
            retval = boinc_mkdir("shared");
            if (retval) {
606
607
608
609
610
611
612
613
614
615
616
                vboxlog_msg("ERROR: couldn't create shared directory: %s.", boincerror(retval));
            }
        }
        if (boinc_file_exists(scratch_dir.c_str())) {
            if (!is_dir(scratch_dir.c_str())) {
                vboxlog_msg("ERROR: 'scratch' exists but is not a directory.");
            }
        } else {
            retval = boinc_mkdir(scratch_dir.c_str());
            if (retval) {
                vboxlog_msg("ERROR: couldn't create scratch directory: %s.", boincerror(retval));
Rom Walton's avatar
Rom Walton committed
617
618
619
620
621
622
            }
        }
    }

    // Copy files to the shared directory
    //
623
    if (pVM->enable_shared_directory && pVM->copy_to_shared.size()) {
624
        for (vector<string>::iterator iter = pVM->copy_to_shared.begin(); iter != pVM->copy_to_shared.end(); ++iter) {
Rom Walton's avatar
Rom Walton committed
625
626
627
628
            string source = *iter;
            string destination = string("shared/") + *iter;
            if (!boinc_file_exists(destination.c_str())) {
                if (!boinc_copy(source.c_str(), destination.c_str())) {
629
                    vboxlog_msg("Successfully copied '%s' to the shared directory.", source.c_str());
Rom Walton's avatar
Rom Walton committed
630
                } else {
631
                    vboxlog_msg("Failed to copy '%s' to the shared directory.", source.c_str());
Rom Walton's avatar
Rom Walton committed
632
633
634
635
636
                }
            }
        }
    }

637
638
    // Configure Instance specific VM Parameters
    //
639
640
    pVM->vm_master_name = "boinc_";
    pVM->image_filename = IMAGE_FILENAME_COMPLETE;
Rom Walton's avatar
Rom Walton committed
641
    if (boinc_is_standalone()) {
642
643
644
        pVM->vm_master_name += "standalone";
        pVM->vm_master_description = "standalone";
        if (pVM->enable_floppyio) {
645
646
647
            sprintf(buf, "%s.%s",
                FLOPPY_IMAGE_FILENAME, FLOPPY_IMAGE_FILENAME_EXTENSION
            );
648
            pVM->floppy_image_filename = buf;
Rom Walton's avatar
Rom Walton committed
649
650
        }
    } else {
651
652
        pVM->vm_master_name += md5_string(std::string(aid.result_name)).substr(0, 16);
        pVM->vm_master_description = aid.result_name;
653
		if (vm_image) {
654
655
656
            sprintf(buf, "%s_%d.%s",
                IMAGE_FILENAME, vm_image, IMAGE_FILENAME_EXTENSION
            );
657
            pVM->image_filename = buf;
658
		}
659
        if (pVM->enable_floppyio) {
660
661
662
663
            sprintf(buf, "%s_%d.%s",
                FLOPPY_IMAGE_FILENAME, aid.slot,
                FLOPPY_IMAGE_FILENAME_EXTENSION
            );
664
            pVM->floppy_image_filename = buf;
Rom Walton's avatar
Rom Walton committed
665
        }
666
    }
667
668
    if (pVM->enable_cache_disk) {
        pVM->cache_disk_filename = CACHE_DISK_FILENAME;
669
    }
670
671
    if (pVM->enable_isocontextualization) {
        pVM->iso_image_filename = ISO_IMAGE_FILENAME;
Rom Walton's avatar
Rom Walton committed
672
673
674
675
676
677
678
    }
    if (aid.ncpus > 1.0 || ncpus > 1.0) {
        if (ncpus) {
            sprintf(buf, "%d", (int)ceil(ncpus));
        } else {
            sprintf(buf, "%d", (int)ceil(aid.ncpus));
        }
679
        pVM->vm_cpu_count = buf;
Rom Walton's avatar
Rom Walton committed
680
    } else {
681
        pVM->vm_cpu_count = "1";
Rom Walton's avatar
Rom Walton committed
682
    }
683
    if (pVM->memory_size_mb > 1.0 || memory_size_mb > 1.0) {
684
685
686
        if (memory_size_mb) {
            sprintf(buf, "%d", (int)ceil(memory_size_mb));
        } else {
687
            sprintf(buf, "%d", (int)ceil(pVM->memory_size_mb));
688
689
        }
    }
Rom Walton's avatar
Rom Walton committed
690
    if (aid.vbox_window && !aid.using_sandbox) {
691
        pVM->headless = false;
Rom Walton's avatar
Rom Walton committed
692
693
694
    }

    // Restore from checkpoint
695
    //
696
697
698
    checkpoint.parse();
    elapsed_time = checkpoint.elapsed_time;
    current_cpu_time = checkpoint.cpu_time;
699
700
    pVM->pf_host_port = checkpoint.webapi_port;
    pVM->rd_host_port = checkpoint.remote_desktop_port;
701
    last_checkpoint_elapsed_time = elapsed_time;
702
    starting_cpu_time = current_cpu_time;
703
    last_checkpoint_cpu_time = current_cpu_time;
Rom Walton's avatar
Rom Walton committed
704
705

    // Should we even try to start things up?
706
    //
707
    if (pVM->job_duration && (elapsed_time > pVM->job_duration)) {
Rom Walton's avatar
Rom Walton committed
708
709
710
        return EXIT_TIME_LIMIT_EXCEEDED;
    }

711
    retval = pVM->run(current_cpu_time > 0);
Rom Walton's avatar
Rom Walton committed
712
    if (retval) {
713
        // All 'failure to start' errors are unrecoverable by default
714
        bool   unrecoverable_error = true;
715
        bool   skip_cleanup = false;
716
        bool   do_dump_hypervisor_logs = false;
717
        string error_reason;
718
        const char*  temp_reason = "";
Rom Walton's avatar
Rom Walton committed
719

720
721
722
723
724
        if (VBOXWRAPPER_ERR_RECOVERABLE == retval) {
            error_reason =
                "    BOINC will be notified that it needs to clean up the environment.\n"
                "    This is a temporary problem and so this job will be rescheduled for another time.\n";
            unrecoverable_error = false;
725
            temp_reason = "VM environment needed to be cleaned up.";
726
        } else if (ERR_NOT_EXITED == retval) {
727
728
729
730
731
            error_reason =
                "   NOTE: VM was already running.\n"
                "    BOINC will be notified that it needs to clean up the environment.\n"
                "    This might be a temporary problem and so this job will be rescheduled for another time.\n";
            unrecoverable_error = false;
732
            temp_reason = "VM environment needed to be cleaned up.";
733
734
        } else if (ERR_INVALID_PARAM == retval) {
            unrecoverable_error = false;
735
            temp_reason = "Please upgrade BOINC to the latest version.";
736
            temp_delay = 86400;
737
        } else if (retval == (int)RPC_S_SERVER_UNAVAILABLE) {
738
739
740
741
742
            error_reason =
                "    VboxSvc crashed while attempting to restore the current snapshot.  This is a critical\n"
                "    operation and this job cannot be recovered.\n";
            skip_cleanup = true;
            retval = ERR_EXEC;
743
744
745
746
747
748
749
        } else if (retval == (int)VBOX_E_INVALID_OBJECT_STATE) {
            error_reason =
                "   NOTE: VM session lock error encountered.\n"
                "    BOINC will be notified that it needs to clean up the environment.\n"
                "    This might be a temporary problem and so this job will be rescheduled for another time.\n";
            unrecoverable_error = false;
            temp_reason = "VM environment needed to be cleaned up.";
750
        } else if (pVM->is_logged_failure_vm_extensions_disabled()) {
751
752
            error_reason =
                "   NOTE: BOINC has detected that your computer's processor supports hardware acceleration for\n"
Rom Walton's avatar
Rom Walton committed
753
                "    virtual machines but the hypervisor failed to successfully launch with this feature enabled.\n"
Rom Walton's avatar
Rom Walton committed
754
                "    This means that the hardware acceleration feature has been disabled in the computer's BIOS.\n"
Rom Walton's avatar
Rom Walton committed
755
                "    Please enable this feature in your computer's BIOS.\n"
Rom Walton's avatar
Rom Walton committed
756
757
                "    Intel calls it 'VT-x'\n"
                "    AMD calls it 'AMD-V'\n"
Rom Walton's avatar
Rom Walton committed
758
                "    More information can be found here: http://en.wikipedia.org/wiki/X86_virtualization\n"
759
                "    Error Code: ERR_CPU_VM_EXTENSIONS_DISABLED\n";
760
            retval = ERR_EXEC;
761
        } else if (pVM->is_logged_failure_vm_extensions_not_supported()) {
762
763
            error_reason =
                "   NOTE: VirtualBox has reported an improperly configured virtual machine. It was configured to require\n"
Rom Walton's avatar
Rom Walton committed
764
                "    hardware acceleration for virtual machines, but your processor does not support the required feature.\n"
765
                "    Please report this issue to the project so that it can be addresssed.\n";
766
        } else if (pVM->is_logged_failure_vm_extensions_in_use()) {
767
768
769
770
            error_reason =
                "   NOTE: VirtualBox hypervisor reports that another hypervisor has locked the hardware acceleration\n"
                "    for virtual machines feature in exclusive mode.\n";
            unrecoverable_error = false;
771
            temp_reason = "Forign VM Hypervisor locked hardware acceleration features.";
772
            temp_delay = 86400;
773
        } else if (pVM->is_logged_failure_host_out_of_memory()) {
774
775
776
            error_reason =
                "   NOTE: VirtualBox has failed to allocate enough memory to start the configured virtual machine.\n"
                "    This might be a temporary problem and so this job will be rescheduled for another time.\n";
Rom Walton's avatar
Rom Walton committed
777
            unrecoverable_error = false;
778
            temp_reason = "VM Hypervisor was unable to allocate enough memory to start VM.";
Rom Walton's avatar
Rom Walton committed
779
        } else {
780
            do_dump_hypervisor_logs = true;
781
        }
Rom Walton's avatar
Rom Walton committed
782

783
784
        if (unrecoverable_error) {
            // Attempt to cleanup the VM and exit.
785
            if (!skip_cleanup) {
786
                pVM->cleanup();
787
            }
788
789

            checkpoint.update(elapsed_time, current_cpu_time);
790
791

            if (error_reason.size()) {
792
                vboxlog_msg("\n%s", error_reason.c_str());
793
794
            }

795
            if (do_dump_hypervisor_logs) {
796
                pVM->dump_hypervisor_logs(true);
797
798
            }

799
800
801
            boinc_finish(retval);
        } else {
            // if the VM is already running notify BOINC about the process ID so it can
Rom Walton's avatar
Rom Walton committed
802
803
            // clean up the environment.  We should be safe to run after that.
            //
804
            if (pVM->vm_pid) {
Rom Walton's avatar
Rom Walton committed
805
                retval = boinc_report_app_status_aux(
806
                    current_cpu_time,
807
                    last_checkpoint_cpu_time,
Rom Walton's avatar
Rom Walton committed
808
                    fraction_done,
809
                    pVM->vm_pid,
Rom Walton's avatar
Rom Walton committed
810
811
812
813
                    bytes_sent,
                    bytes_received
                );
            }
814
815
 
            // Give the BOINC API time to report the pid to BOINC.
816
            //
817
818
            boinc_sleep(5.0);

819
            if (error_reason.size()) {
820
                vboxlog_msg("\n%s", error_reason.c_str());
821
822
            }

823
            // Exit and let BOINC clean up the rest.
824
            //
Rom Walton's avatar
Rom Walton committed
825
826
827
828
            boinc_temporary_exit(temp_delay, temp_reason);
        }
    }

829
830
    // Report the VM pid to BOINC so BOINC can deal with it when needed.
    //
831
    vboxlog_msg("Reporting VM Process ID to BOINC.");
832
    retval = boinc_report_app_status_aux(
833
        current_cpu_time,
834
        last_checkpoint_cpu_time,
835
        fraction_done,
836
        pVM->vm_pid,
837
838
839
840
        bytes_sent,
        bytes_received
    );

841
842
843
844
    // Wait for up to 5 minutes for the VM to switch states.
    // A system under load can take a while.
    // Since the poll function can wait for up to 60 seconds
    // to execute a command we need to make this time based instead
845
    // of iteration based.
846
    //
847
848
    timeout = dtime() + 300;
    do {
849
850
        pVM->poll(false);
        if (pVM->online && !pVM->restoring) break;
851
852
        boinc_sleep(1.0);
    } while (timeout >= dtime());
853
854

    // Lower the VM process priority after it has successfully brought itself online.
855
    //
856
    pVM->lower_vm_process_priority();
857
858

    // Log our current state 
859
    pVM->poll(true);
860
861

    // Did we timeout?
862
    if (!pVM->online && (timeout <= dtime())) {
863
864
        vboxlog_msg("NOTE: VM failed to enter an online state within the timeout period.");
        vboxlog_msg("  This might be a temporary problem and so this job will be rescheduled for another time.");
865
866
        pVM->reset_vm_process_priority();
        pVM->poweroff();
867
        pVM->dump_hypervisor_logs(true);
868
869
870
        boinc_temporary_exit(86400,
            "VM Hypervisor failed to enter an online state in a timely fashion."
        );
871
872
    }

873
    set_floppy_image(aid, *pVM);
874
875
    report_web_graphics_url(*pVM);
    report_remote_desktop_info(*pVM);
876
877
878
    checkpoint.webapi_port = pVM->pf_host_port;
    checkpoint.remote_desktop_port = pVM->rd_host_port;
    checkpoint.update(elapsed_time, current_cpu_time);
Rom Walton's avatar
Rom Walton committed
879

880
881
882
    // Force throttling on our first pass through the loop
    boinc_status.reread_init_data_file = true;

Rom Walton's avatar
Rom Walton committed
883
884
    while (1) {
        // Begin stopwatch timer
885
        stopwatch_starttime = dtime();
886
        loop_iteration += 1;
Rom Walton's avatar
Rom Walton committed
887
888

        // Discover the VM's current state
889
890
891
892
893
894
895
        retval = pVM->poll();
        if (retval) {
            vboxlog_msg("ERROR: Vboxwrapper lost communication with VirtualBox, rescheduling task for a later time.");
            pVM->reset_vm_process_priority();
            pVM->poweroff();
            boinc_temporary_exit(86400, "VM job unmanageable, restarting later.");
        }
Rom Walton's avatar
Rom Walton committed
896

897
898
899
900
        // Write updates for the graphics application's use
        if (pVM->enable_graphics_support) {
            boinc_write_graphics_status(current_cpu_time, elapsed_time, fraction_done);
        }
901

Rom Walton's avatar
Rom Walton committed
902
        if (boinc_status.no_heartbeat || boinc_status.quit_request) {
903
            pVM->reset_vm_process_priority();
904
905
906
907
908
909
910
911
912
913
            if (pVM->enable_vm_savestate_usage) {
                retval = pVM->create_snapshot(elapsed_time);
                if (!retval) {
                    checkpoint.update(elapsed_time, current_cpu_time);
                    boinc_checkpoint_completed();
                }
                pVM->stop();
            } else {
                pVM->poweroff();
            }
914
            boinc_temporary_exit(86400);
Rom Walton's avatar
Rom Walton committed
915
916
        }
        if (boinc_status.abort_request) {
917
918
919
            pVM->reset_vm_process_priority();
            pVM->cleanup();
            pVM->dump_hypervisor_logs(true);
Rom Walton's avatar
Rom Walton committed
920
921
            boinc_finish(EXIT_ABORTED_BY_CLIENT);
        }
922
        if (completion_file_exists(*pVM)) {
923
            vboxlog_msg("VM Completion File Detected.");
924
            read_completion_file_info(vm_exit_code, is_notice, message, *pVM);
925
            if (message.size()) {
926
                vboxlog_msg("VM Completion Message: %s.", message.c_str());
927
            }
928
929
            pVM->reset_vm_process_priority();
            pVM->cleanup();
930
            if (is_notice) {
931
932
933
934
                boinc_finish_message(vm_exit_code, message.c_str(), is_notice);
            } else {
                boinc_finish(vm_exit_code);
            }
935
        }
936
        if (temporary_exit_file_exists(*pVM)) {
937
            vboxlog_msg("VM Temporary Exit File Detected.");
938
            read_temporary_exit_file_info(temp_delay, is_notice, message, *pVM);
939
            if (message.size()) {