repair mongodb after usafe shutdown
171354
def repair_mongo(name, dbpath):
    """"""
    log_file = os.path.join(dbpath, 'mongod.log')
    cmd = [name, "--dbpath", dbpath, "--logpath", log_file, "--logappend",
           "--repair"]
    proc = subprocess.Popen(
        cmd, universal_newlines=True,
        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    timeout = 45
    t_start = time.time()
    while time.time() - t_start < timeout:
        line = str(proc.stdout.readline())
        logger.info("repair output: %s" % (line,))
        return_code = proc.poll()
        if return_code is not None:
            if return_code:
                raise Exception("mongod --repair failed with exit code %s, "
                                "check log file: %s" % (return_code, log_file))
            # Success when poll() returns 0
            return
        time.sleep(1)
    proc.terminate()
    raise Exception("mongod --repair failed to exit after %s seconds, "
                    "check log file: %s" % (timeout, log_file))
1186369
def reset_db():
    
    logger = get_logger(PROCESS_SCHEDULER)
    logger.info('Starting *scheduler* DB reset')

    ds = ds_manager.ds_factory(logger)
    ds._db_client.drop_database(settings.settings['mongo_db_name'])
    logger.info('*scheduler* db has been dropped')

    connection = ds.connection(COLLECTION_MANAGED_PROCESS)
    connection.create_index([(PROCESS_NAME, pymongo.ASCENDING)], unique=True)

    connection = ds.connection(COLLECTION_FREERUN_PROCESS)
    connection.create_index([(PROCESS_NAME, pymongo.ASCENDING), (ENTRY_NAME, pymongo.ASCENDING)], unique=True)

    connection = ds.connection(COLLECTION_UNIT_OF_WORK)
    connection.create_index([(PROCESS_NAME, pymongo.ASCENDING),
                             (TIMEPERIOD, pymongo.ASCENDING),
                             (START_ID, pymongo.ASCENDING),
                             (END_ID, pymongo.ASCENDING)], unique=True)

    connection = ds.connection(COLLECTION_LOG_RECORDING)
    connection.create_index([(PARENT_OBJECT_ID, pymongo.ASCENDING)], unique=True)

    # expireAfterSeconds: <int> Used to create an expiring (TTL) collection.
    # MongoDB will automatically delete documents from this collection after <int> seconds.
    # The indexed field must be a UTC datetime or the data will not expire.
    ttl_seconds = settings.settings['db_log_ttl_days'] * 86400     # number of seconds for TTL
    connection.create_index(CREATED_AT, expireAfterSeconds=ttl_seconds)

    for collection_name in [COLLECTION_JOB_HOURLY, COLLECTION_JOB_DAILY,
                            COLLECTION_JOB_MONTHLY, COLLECTION_JOB_YEARLY]:
        connection = ds.connection(collection_name)
        connection.create_index([(PROCESS_NAME, pymongo.ASCENDING), (TIMEPERIOD, pymongo.ASCENDING)], unique=True)

    # reset Synergy Flow tables
    db_manager.reset_db()
    logger.info('*scheduler* db has been recreated')
264703
function MongoDB () {
  var self = this;

  self.klass = MongoDB;
  self.klass.super_.apply(self, arguments);
}
558514
function configure(opts, done) {
  delete opts._;

  opts = defaults(opts, {
    topology: process.env.MONGODB_TOPOLOGY || 'standalone'
  });

  opts = defaults(opts, {
    name: opts.topology
  });

  opts = defaults(opts, {
    logpath: untildify(
      process.env.MONGODB_LOGPATH ||
        format('~/.mongodb/runner/%s.log', opts.name)
    ),
    pidpath: untildify(process.env.MONGODB_PIDPATH || '~/.mongodb/runner/pid'),
    port: process.env.MONGODB_PORT || 27017,
    mongodBin: process.env.MONGOD_BIN || 'mongod',
    mongosBin: process.env.MONGOS_BIN || 'mongos',
    storageEngine: process.env.MONGODB_STORAGE_ENGINE,
    auth_mechanism: process.env.MONGODB_AUTH_MECHANISM || 'none',
    purge: process.env.MONGODB_PURGE || true
  });

  // MongoDB < 3.0 doesn't understand the storageEngine argument and
  // will fail to start if provided!
  if (opts.version < '3.0') {
    delete opts.storageEngine;
  }

  if (opts.topology === 'replicaset') {
    opts = defaults(opts, {
      arbiters: process.env.MONGODB_ARBITERS || 0,
      secondaries: process.env.MONGODB_SECONDARIES || 2,
      passives: process.env.MONGODB_PASSIVES || 0
    });
  }

  if (opts.topology === 'cluster') {
    opts = defaults(opts, {
      shards: process.env.MONGODB_SHARDS || 1, // -> replsets
      routers: process.env.MONGODB_ROUTERS || 1, // -> mongoses
      configs: process.env.MONGODB_CONFIGS || 1,
      shardPort: process.env.MONGODB_SHARDS_PORT || 31000, // -> replsetStartPort
      configPort: process.env.MONGODB_CONFIGS_PORT || 35000, // -> configStartPort
      arbiters: process.env.MONGODB_ARBITERS || 0,
      secondaries: process.env.MONGODB_SECONDARIES || 2,
      passives: process.env.MONGODB_PASSIVES || 0
    });
  }

  debug('Ready to process spec', opts);

  if (opts.action === 'stop') {
    return done();
  }
  async.series(
    [
      mkdirp.bind(null, opts.pidpath),
      getDbPath.bind(null, opts),
      createLogsDirectory.bind(null, opts)
    ],
    done
  );
}
1225078
function(cb) {
		modules.logger.log("shutdown","ISNode System > Attempting to close all open modules.");
		var modCount = 0;
		var stdModCount = 0;
		var interfaceModCount = 0;
		var counter = 0;
		if(config.loader.timeouts.closeModules)
			var timeout = config.loader.timeouts.closeModules
		else
			var timeout = 2000;
		var timeoutTimer = 0;

		Object.keys(modules).forEach(function(key) { stdModCount ++; });
		stdModCount = stdModCount - 1;
		Object.keys(modules.interfaces).forEach(function(key) { interfaceModCount ++; });
		modCount = stdModCount + interfaceModCount;

		var interval = setInterval(function(){
	    	if(counter >= (modCount - 1)) {
	    		if(modules.logger.log) {
	    			modules.logger.log("shutdown","ISNode System > Modules all shutdown successfully ("+counter+"/"+(modCount - 1)+").");
	    		} else {
					var currentDate = new Date();
					currentDate = currentDate.toISOString();
					console.log(currentDate + "(shutdown) ISNode System > Modules all shutdown successfully ("+counter+"/"+(modCount - 1)+").");
	    		}
			    clearInterval(interval);
			    cb(null, {success:true, message: "Modules all shutdown successfully"});
			    return;	
	    	}
	    	if(timeoutTimer > timeout) {
	    		if(modules.logger.log) {
	    			modules.logger.log("shutdown","ISNode System > Module shutdown timed out ("+counter+"/"+(modCount - 1)+" Closed Successfully).");
	    		} else {
					var currentDate = new Date();
					currentDate = currentDate.toISOString();
					console.log(currentDate + "(shutdown) ISNode System > Module shutdown timed out ("+counter+"/"+(modCount - 1)+" Closed Successfully).");
	    		}
	    		clearInterval(interval);
	    		var error = { message: "Module shutdown timed out" };
	    		cb(error, null);
	    		return;
	    	}
	    	timeoutTimer += 500;
	    }, 500);

		process.nextTick(function(){
			isnode.on("module-shut-down", function(){
				counter ++;
			});
			isnode.emit("shutdown", "Shutdown All Modules");
	    	return;
    	});
	}
66714
def configure_mongodb(self):
        
        self._display_info("Trying default configuration")

        host = "localhost"
        database_name = "INGInious"

        should_ask = True
        if self.try_mongodb_opts(host, database_name):
            should_ask = self._ask_boolean(
                "Successfully connected to MongoDB. Do you want to edit the configuration anyway?", False)
        else:
            self._display_info("Cannot guess configuration for MongoDB.")

        while should_ask:
            self._display_question(
                "Please enter the MongoDB host. If you need to enter a password, here is the syntax:")
            self._display_question("mongodb://USERNAME:PASSWORD@HOST:PORT/AUTHENTIFICATION_DATABASE")
            host = self._ask_with_default("MongoDB host", host)
            database_name = self._ask_with_default("Database name", database_name)
            if not self.try_mongodb_opts(host, database_name):
                if self._ask_boolean("Cannot connect to MongoDB. Would you like to continue anyway?", False):
                    break
            else:
                self._display_info("Successfully connected to MongoDB")
                break

        return {"mongo_opt": {"host": host, "database": database_name}}
362025
protected void cleanupOnExit() {
        String threadName = "Shutdown Hook Deletion Thread for Temporary DB " + dataDir.toString();
        final DB db = this;
        Runtime.getRuntime().addShutdownHook(new Thread(threadName) {

            @Override
            public void run() {
                // ManagedProcess DestroyOnShutdown ProcessDestroyer does
                // something similar, but it shouldn't hurt to better be save
                // than sorry and do it again ourselves here as well.
                try {
                    // Shut up and don't log if it was already stop() before
                    if (mysqldProcess != null && mysqldProcess.isAlive()) {
                        logger.info("cleanupOnExit() ShutdownHook now stopping database");
                        db.stop();
                    }
                } catch (ManagedProcessException e) {
                    logger.warn("cleanupOnExit() ShutdownHook: An error occurred while stopping the database", e);
                }

                if (dataDir.exists() && (configuration.isDeletingTemporaryBaseAndDataDirsOnShutdown() && Util.isTemporaryDirectory(dataDir.getAbsolutePath()))) {
                    logger.info("cleanupOnExit() ShutdownHook quietly deleting temporary DB data directory: " + dataDir);
                    FileUtils.deleteQuietly(dataDir);
                }
                if (baseDir.exists() && (configuration.isDeletingTemporaryBaseAndDataDirsOnShutdown() && Util.isTemporaryDirectory(dataDir.getAbsolutePath()))) {
                    logger.info("cleanupOnExit() ShutdownHook quietly deleting temporary DB base directory: " + baseDir);
                    FileUtils.deleteQuietly(baseDir);
                }
            }
        });
    }
643628
def reloadCollections(self) :
        ""
        r = self.connection.session.get(self.collectionsURL)
        data = r.json()
        if r.status_code == 200 :
            self.collections = {}

            for colData in data["result"] :
                colName = colData['name']
                if colData['isSystem'] :
                    colObj = COL.SystemCollection(self, colData)
                else :
                    try :
                        colClass = COL.getCollectionClass(colName)
                        colObj = colClass(self, colData)
                    except KeyError :
                        if colData["type"] == CONST.COLLECTION_EDGE_TYPE :
                            colObj = COL.Edges(self, colData)
                        elif colData["type"] == CONST.COLLECTION_DOCUMENT_TYPE :
                            colObj = COL.Collection(self, colData)
                        else :
                            print(("Warning!! Collection of unknown type: %d, trying to load it as Collection nonetheless." % colData["type"]))
                            colObj = COL.Collection(self, colData)

                self.collections[colName] = colObj
        else :
            raise UpdateError(data["errorMessage"], data)
822686
def refresh_db(root=None, **kwargs):
    '''
    
    '''
    # Remove rtag file to keep multiple refreshes from happening in pkg states
    salt.utils.pkg.clear_rtag(__opts__)
    cmd = ['pacman', '-Sy']

    if root is not None:
        cmd.extend(('-r', root))

    ret = {}
    call = __salt__['cmd.run_all'](cmd,
                                   output_loglevel='trace',
                                   env={'LANG': 'C'},
                                   python_shell=False)
    if call['retcode'] != 0:
        comment = ''
        if 'stderr' in call:
            comment += ': ' + call['stderr']
        raise CommandExecutionError(
            'Error refreshing package database' + comment
        )
    else:
        out = call['stdout']

    for line in salt.utils.itertools.split(out, '\n'):
        if line.strip().startswith('::'):
            continue
        if not line:
            continue
        key = line.strip().split()[0]
        if 'is up to date' in line:
            ret[key] = False
        elif 'downloading' in line:
            key = line.strip().split()[1].split('.')[0]
            ret[key] = True
    return ret
1086133
def reset(c7n_async=None):
    
    click.echo('Delete db? Are you Sure? [yn] ', nl=False)
    c = click.getchar()
    click.echo()
    if c == 'y':
        click.echo('Wiping database')
        worker.connection.flushdb()
    elif c == 'n':
        click.echo('Abort!')
    else:
        click.echo('Invalid input :(')
688131
public synchronized void shutdown() {
		if (instances.size() > 0) {
			OLogManager.instance().debug(null,
					"Found %d databases opened during OrientDB shutdown. Assure to always close database instances after usage",
					instances.size());

			for (ODatabaseComplex<?> db : new HashSet<ODatabaseComplex<?>>(instances.keySet())) {
				if (db != null && !db.isClosed()) {
					db.close();
				}
			}
		}
	}
1159257
def dbRestore(self, db_value, context=None):
        
        if db_value is not None:
            try:
                return rest.unjsonify(db_value)
            except StandardError:
                log.exception('Failed to restore json')
                raise orb.errors.DataStoreError('Failed to restore json.')
        else:
            return db_value
1328196
def _merge_legacy(run, full_result, result, index):
    
    affected = result.get('n', 0)

    errmsg = result.get("errmsg", result.get("err", ""))
    if errmsg:
        # wtimeout is not considered a hard failure in
        # MongoDB 2.6 so don't treat it like one here.
        if result.get("wtimeout"):
            error_doc = {'errmsg': errmsg, 'code': _WRITE_CONCERN_ERROR}
            full_result['writeConcernErrors'].append(error_doc)
        else:
            code = result.get("code", _UNKNOWN_ERROR)
            error = _make_error(run.index(index), code, errmsg, run.ops[index])
            if "errInfo" in result:
                error["errInfo"] = result["errInfo"]
            full_result["writeErrors"].append(error)
            return
    if run.op_type == _INSERT:
        full_result['nInserted'] += 1
    elif run.op_type == _UPDATE:
        if "upserted" in result:
            doc = {_UINDEX: run.index(index), _UID: result["upserted"]}
            full_result["upserted"].append(doc)
            full_result['nUpserted'] += affected
        # Versions of MongoDB before 2.6 don't return the _id for an
        # upsert if _id is not an ObjectId.
        elif result.get("updatedExisting") is False and affected == 1:
            op = run.ops[index]
            # If _id is in both the update document *and* the query spec
            # the update document _id takes precedence.
            _id = op['u'].get('_id', op['q'].get('_id'))
            doc = {_UINDEX: run.index(index), _UID: _id}
            full_result["upserted"].append(doc)
            full_result['nUpserted'] += affected
        else:
            full_result['nMatched'] += affected

    elif run.op_type == _DELETE:
        full_result['nRemoved'] += affected
1099922
@POST
    @Path("{uuid}/repair")
    public Result repair(@PathParam("uuid") String uuid) {
        MigrationFeature.checkMigrationId(uuid);
        if (failMigrations.isEmpty()) {
            throw new NotFoundException();
        }
        for (String dbName : failMigrations.keySet()) {
            MigrationFail fail = failMigrations.get(dbName);
            fail.flyway.repair();
            fail.migration.persist();
            fail.migration.reset();
        }
        failMigrations.clear();
        return Result.success();
    }
839187
@Override
	public void shutdown() {
		if (collection != null) {
			try {
				collection.shutdown();
			} catch (IOException e) {
				LOGGER.severe("FAILED collection shutdown" + e.getMessage());
			}
		}
		if (exclusionFactory != null) {
			exclusionFactory.shutdown();
		}
	}
1107226
def rescue_connection_failure(max_retries = 40)
      success = false
      retries = 0
      while !success
        begin
          yield
          success = true
        rescue mongo_error_type => e
          raise e if (retries += 1) >= max_retries
          sleep 0.25
        end
      end
    end
230538
def db_repair(name,
              table=None,
              **connection_args):
    '''
    
    '''
    ret = []
    if table is None:
        # we need to repair all tables
        tables = db_tables(name, **connection_args)
        for table in tables:
            log.info('Repairing table \'%s\' in db \'%s\'..', name, table)
            ret.append(__repair_table(name, table, **connection_args))
    else:
        log.info('Repairing table \'%s\' in db \'%s\'..', name, table)
        ret = __repair_table(name, table, **connection_args)
    return ret