TODO: Dynamically load UDFs
2415
private void registerUDFs() {
    DateParseUDF.register(this.mHiveContext);
    DateTimeExtractUDF.register(this.mHiveContext);
    DateUDF.registerUDFs(this.mHiveContext);
  }
29624
def record(self):
        # type: () -> bytes
        '''
        
        '''
        if not self._initialized:
            raise pycdlibexception.PyCdlibInternalError('UDF Logical Volume Implementation Use not initialized')

        return struct.pack(self.FMT, self.impl_id.record(),
                           self.num_files, self.num_dirs,
                           self.min_udf_read_revision,
                           self.min_udf_write_revision,
                           self.max_udf_write_revision) + self.impl_use
754152
List<LocatedBlock> corruptBlocksInFile(
    DistributedFileSystem fs, String uriPath, FileStatus stat)
  throws IOException {
    List<LocatedBlock> corrupt = new LinkedList<LocatedBlock>();
    LocatedBlocks locatedBlocks = fs.getClient().namenode.getBlockLocations(
      uriPath, 0, stat.getLen());
    for (LocatedBlock b: locatedBlocks.getLocatedBlocks()) {
      if (b.isCorrupt() || 
         (b.getLocations().length == 0 && b.getBlockSize() > 0)) {
        LOG.info("Adding bad block for file " + uriPath);
        corrupt.add(b);
      }
    }
    return corrupt;
  }
581372
HdfsFileStatus getHdfsFileInfo(String src) {
    String srcs = normalizePath(src);
    byte[][] components = INodeDirectory.getPathComponents(srcs);
    readLock();
    try {
      INode targetNode = rootDir.getNode(components);
      if (targetNode == null) {
        return null;
      }
      else {
        return createHdfsFileStatus(HdfsFileStatus.EMPTY_NAME, targetNode);
      }
    } finally {
      readUnlock();
    }
  }
716173
function(rows){
			if (rows.length === 0)
				return onDone();
			db.takeConnectionFromPool(function (conn) {
				async.eachSeries(
					rows,
					function (row, cb) {
						breadcrumbs.add("--------------- archiving uncovered unit " + row.unit);
						storage.readJoint(conn, row.unit, {
							ifNotFound: function () {
								throw Error("nonserial unit not found?");
							},
							ifFound: function (objJoint) {
								mutex.lock(["write"], function(unlock){
									var arrQueries = [];
									conn.addQuery(arrQueries, "BEGIN");
									archiving.generateQueriesToArchiveJoint(conn, objJoint, 'uncovered', arrQueries, function(){
										conn.addQuery(arrQueries, "COMMIT");
										async.series(arrQueries, function(){
											breadcrumbs.add("------- done archiving "+row.unit);
											var parent_units = storage.assocUnstableUnits[row.unit].parent_units;
											storage.forgetUnit(row.unit);
											parent_units.forEach(function(parent_unit){
												if (!storage.assocUnstableUnits[parent_unit]) // the parent is already stable
													return;
												var bHasChildren = false;
												for (var unit in storage.assocUnstableUnits){
													var o = storage.assocUnstableUnits[unit];
													if (o.parent_units.indexOf(parent_unit) >= 0)
														bHasChildren = true;
												}
												if (!bHasChildren)
													storage.assocUnstableUnits[parent_unit].is_free = 1;
											});
											unlock();
											cb();
										});
									});
								});
							}
						});
					},
					function () {
						conn.query(
							"UPDATE units SET is_free=1 WHERE is_free=0 AND is_stable=0 \n\
							AND (SELECT 1 FROM parenthoods WHERE parent_unit=unit LIMIT 1) IS NULL",
							function () {
								conn.release();
								if (rows.length > 0)
									return purgeUncoveredNonserialJoints(false, onDone); // to clean chains of bad units
								onDone();
							}
						);
					}
				);
			});
		}
103302
function(T, SA, fs, n, k, isbwt) {
        var C, B, RA;
        var i, j, b, c, m, p, q, name, pidx = 0, newfs;
        var c0, c1;
        var flags = 0;

        // allocate temporary storage [CSA]
        if (k <= 256) {
            C = Util.makeS32Buffer(k);
            if (k <= fs) { B = SA.subarray(n + fs - k); flags = 1; }
            else { B = Util.makeS32Buffer(k); flags = 3; }
        } else if (k <= fs) {
            C = SA.subarray(n + fs - k);
            if (k <= (fs - k)) { B = SA.subarray(n + fs - k * 2); flags = 0; }
            else if (k <= 1024) { B = Util.makeS32Buffer(k); flags = 2; }
            else { B = C; flags = 8; }
        } else {
            C = B = Util.makeS32Buffer(k);
            flags = 4 | 8;
        }

        /* stage 1: reduce the problem by at least 1/2
           sort all the LMS-substrings */
        getCounts(T, C, n, k);
        getBuckets(C, B, k, true); /* find ends of buckets */
        for (i = 0; i < n; i++) { SA[i] = 0; }
        b = -1; i = n - 1; j = n; m = 0; c0 = T[n - 1];
        do { c1 = c0; } while ((--i >= 0) && ((c0 = T[i]) >= c1));
        for (; i >= 0 ;) {
            do { c1 = c0; } while ((--i >= 0) && ((c0 = T[i]) <= c1));
            if ( i >= 0 ) {
                if ( b >= 0 ) { SA[b] = j; }
                b = --B[c1];
                j = i;
                ++m;
                do { c1 = c0; } while ((--i >= 0) && ((c0 = T[i]) >= c1));
            }
        }

        if (m > 1) {
            LMSsort(T, SA, C, B, n, k);
            name = LMSpostproc(T, SA, n, m);
        } else if (m === 1) {
            SA[b] = j + 1;
            name = 1;
        } else {
            name = 0;
        }

        /* stage 2: solve the reduced problem
           recurse if names are not yet unique */
        if(name < m) {
            if((flags & 4) !== 0) { C = null; B = null; }
            if((flags & 2) !== 0) { B = null; }
            newfs = (n + fs) - (m * 2);
            if((flags & (1 | 4 | 8)) === 0) {
                if((k + name) <= newfs) { newfs -= k; }
                else { flags |= 8; }
            }
            ASSERT( (n >>> 1) <= (newfs + m) );
            for (i = m + (n >>> 1) - 1, j = m * 2 + newfs - 1; m <= i; i--) {
                if(SA[i] !== 0) { SA[j--] = SA[i] - 1; }
            }
            RA = SA.subarray(m + newfs);
            SA_IS(RA, SA, newfs, m, name, false);
            RA = null;

            i = n - 1; j = m * 2 - 1; c0 = T[n - 1];
            do { c1 = c0; } while ((--i >= 0) && ((c0 = T[i]) >= c1));
            for (; i >= 0 ;) {
                do { c1 = c0; } while ((--i >= 0) && ((c0 = T[i]) <= c1));
                if ( i >= 0 ) {
                    SA[j--] = i + 1;
                    do { c1 = c0; } while ((--i >= 0) && ((c0 = T[i]) >= c1));
                }
            }

            for (i = 0; i < m; i++) { SA[i] = SA[m + SA[i]]; }
            if((flags & 4) !== 0) { C = B = Util.makeS32Buffer(k); }
            if((flags & 2) !== 0) { B = Util.makeS32Buffer(k); }
        }

        /* stage 3: induce the result for the original problem */
        if((flags & 8) !== 0) { getCounts(T, C, n, k); }
        /* put all left-most S characters into their buckets */
        if (m > 1) {
            getBuckets(C, B, k, true); /* find ends of buckets */
            i = m - 1; j = n; p = SA[m - 1]; c1 = T[p];
            do {
                q = B[c0 = c1];
                while (q < j) { SA[--j] = 0; }
                do {
                    SA[--j] = p;
                    if(--i < 0) { break; }
                    p = SA[i];
                } while((c1 = T[p]) === c0);
            } while (i >= 0 );
            while ( j > 0 ) { SA[--j] = 0; }
        }
        if (!isbwt) { induceSA(T, SA, C, B, n, k); }
        else { pidx = computeBWT(T, SA, C, B, n, k); }
        C = null; B = null;
        return pidx;
    }
267267
@Override
	public void prepare() throws Exception {
		final TaskConfig config = this.taskContext.getTaskConfig();
		this.strategy = config.getDriverStrategy();
		
		if (strategy == DriverStrategy.ALL_GROUP_COMBINE) {
			if (!(this.taskContext.getStub() instanceof GenericCombine)) {
				throw new Exception("Using combiner on a UDF that does not implement the combiner interface " + GenericCombine.class.getName());
			}
		}
		else if (strategy != DriverStrategy.ALL_GROUP_REDUCE) {
			throw new Exception("Unrecognized driver strategy for AllGroupReduce driver: " + config.getDriverStrategy().name());
		}
		this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
		this.input = this.taskContext.getInput(0);
	}
479255
private List<GridFSDBFile> getGFSDBFiles(BasicDBObject mongoQuery, BasicDBObject sort, String collectionName,
            int maxResult, int firstResult)
    {
        KunderaGridFS gfs = new KunderaGridFS(mongoDb, collectionName);
        return gfs.find(mongoQuery, sort, firstResult, maxResult);
    }
695721
private Optional<UfsStatus> syncPersistDirectory(InodeDirectoryView dir)
      throws FileDoesNotExistException, IOException, InvalidPathException {
    AlluxioURI uri = getPath(dir);
    MountTable.Resolution resolution = mMountTable.resolve(uri);
    String ufsUri = resolution.getUri().toString();
    try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
      UnderFileSystem ufs = ufsResource.get();
      MkdirsOptions mkdirsOptions =
          MkdirsOptions.defaults(ServerConfiguration.global()).setCreateParent(false)
          .setOwner(dir.getOwner()).setGroup(dir.getGroup()).setMode(new Mode(dir.getMode()));
      if (!ufs.mkdirs(ufsUri, mkdirsOptions)) {
        // Directory might already exist. Try loading the status from ufs.
        UfsStatus status;
        try {
          status = ufs.getStatus(ufsUri);
        } catch (Exception e) {
          throw new IOException(String.format("Cannot create or load UFS directory %s: %s.",
              ufsUri, e.toString()), e);
        }
        if (status.isFile()) {
          throw new InvalidPathException(String.format(
              "Error persisting directory. A file exists at the UFS location %s.", ufsUri));
        }
        return Optional.of(status);
      }
    }
    return Optional.empty();
  }
629108
private synchronized void load() {
    Map<String, HadoopServer> map = new TreeMap<String, HadoopServer>();
    for (File file : saveDir.listFiles()) {
      try {
        HadoopServer server = new HadoopServer(file);
        map.put(server.getLocationName(), server);

      } catch (Exception exn) {
        System.err.println(exn);
      }
    }
    this.servers = map;
  }
8539
function() {
         //copy to make real Array
         var args = Array.prototype.slice.call( arguments );
         var action = args.shift();
         args = deduceArgs(args);
         var txn = args[1].txn, data = args[0], callback = args[1].callback,
            comment = args[1].comment, override = args[1].override,
            fileinput = args[1].fileinput;
         var commitNow = false;
         // set changedOnly default to true unless action == replace
         var changedOnly = typeof args[1].changedOnly !== 'undefined'
                              ? args[1].changedOnly : action !== 'replace';
         if (!txn) {
             txn = this.data('currentTxn');
             if (!txn) {
                 txn = new Txn(args[1].url || $.db.url);
                 commitNow = true;
             } else if (args[1].url) {
               konsole.assert(txn.url ===  args[1].url);
             }
         }
      try {
         //konsole.log('execute', action, data, callback);
         var requestIds = [];
         if (data) {
            if (action == 'query') {
                if (!data.conditions) {
                    data = { conditions : data };
                }
                //assert data.query;
                var thisid = this.attr('itemid');
                if (thisid) {
                    if (!data.bindvars) {
                        data.bindvars = {}
                    }
                    data.bindvars['this'] =  thisid;
                }
            } else if (!data[_IDkey] && this.attr('itemid')) {
                data[_IDkey] = this.attr('itemid');
            }
            requestIds = [txn.execute(action, data, callback, this.length ? this[0] : null)];
            if (fileinput)
             txn._addFileInput(fileinput, data, requestIds[0]);
         } else {
            konsole.assert(!fileinput);
            requestIds = this.map(function() {
                var obj = bindElement(this, false, undefined, changedOnly);
                if (override) {
                  if (typeof override == 'function')
                    obj = override(obj);
                  else
                    $.extend(obj[0], override);
                }

                var requestid = txn.execute(action, obj, callback, this);
                $(this).find('[data-dbmethod]').each(function() {
                  txn._addFileInput(this, obj, requestid);
                });
                //konsole.log('about to', action, 'obj', obj);
                return requestid;
            }).get();
         }
         if (comment) { //XXX this is all kind of hacky
            comment = comment.replace('$new0', '$new'+requestIds[0]);
            if (txn.txnComment)
                txn.txnComment += ' and ' + comment;
            else
                txn.txnComment = comment;
         }
       } catch (err) {
         // invoke callback if it hasn't already been bound
         // (if we have requestIds then it has)
         var rollbackCB = null;
         if (callback && !requestIds.length) {
           var elem = this;
           rollbackCB = function(response) {
              // note: if callback returns false event bubbling is stopped
              return callback.call(elem, null, response.error);
           };
         }
         this.dbRollback(rollbackCB, err, txn);
         return this;
       }
         if (commitNow)
            txn.commit(null, this);
         return this;
     }
617775
def record(self):
        # type: () -> bytes
        '''
        
        '''
        if not self._initialized:
            raise pycdlibexception.PyCdlibInternalError('UDF Long AD not initialized')

        return struct.pack(self.FMT, self.extent_length, self.log_block_num,
                           self.part_ref_num, self.impl_use)
1131543
def dbus_readBytesTwoFDs(self, fd1, fd2, byte_count):
        
        result = bytearray()
        for fd in (fd1, fd2):
            f = os.fdopen(fd, 'rb')
            result.extend(f.read(byte_count))
            f.close()
        return result
472574
function (name, data, o) {
      // be sure sub folders exist
      var parent = parentFolder(name), dataType = JSZip.utils.getTypeOf(data);
      if (parent) {
         folderAdd.call(this, parent);
      }

      o = prepareFileAttrs(o);

      if (o.dir || data === null || typeof data === "undefined") {
         o.base64 = false;
         o.binary = false;
         data = null;
      } else if (dataType === "string") {
         if (o.binary && !o.base64) {
            // optimizedBinaryString == true means that the file has already been filtered with a 0xFF mask
            if (o.optimizedBinaryString !== true) {
               // this is a string, not in a base64 format.
               // Be sure that this is a correct "binary string"
               data = JSZip.utils.string2binary(data);
            }
         }
      } else { // arraybuffer, uint8array, ...
         o.base64 = false;
         o.binary = true;

         if (!dataType && !(data instanceof JSZip.CompressedObject)) {
            throw new Error("The data of '" + name + "' is in an unsupported format !");
         }

         // special case : it's way easier to work with Uint8Array than with ArrayBuffer
         if (dataType === "arraybuffer") {
            data = JSZip.utils.transformTo("uint8array", data);
         }
      }

      return this.files[name] = new ZipObject(name, data, o);
   }
1184375
void process(byte ttfAfm[], boolean preload) throws DocumentException, IOException {
        tables = new HashMap();
        
        try {
            if (ttfAfm == null)
                rf = new RandomAccessFileOrArray(fileName, preload, Document.plainRandomAccess);
            else
                rf = new RandomAccessFileOrArray(ttfAfm);
            if (ttcIndex.length() > 0) {
                int dirIdx = Integer.parseInt(ttcIndex);
                if (dirIdx < 0)
                    throw new DocumentException("The font index for " + fileName + " must be positive.");
                String mainTag = readStandardString(4);
                if (!mainTag.equals("ttcf"))
                    throw new DocumentException(fileName + " is not a valid TTC file.");
                rf.skipBytes(4);
                int dirCount = rf.readInt();
                if (dirIdx >= dirCount)
                    throw new DocumentException("The font index for " + fileName + " must be between 0 and " + (dirCount - 1) + ". It was " + dirIdx + ".");
                rf.skipBytes(dirIdx * 4);
                directoryOffset = rf.readInt();
            }
            rf.seek(directoryOffset);
            int ttId = rf.readInt();
            if (ttId != 0x00010000 && ttId != 0x4F54544F)
                throw new DocumentException(fileName + " is not a valid TTF or OTF file.");
            int num_tables = rf.readUnsignedShort();
            rf.skipBytes(6);
            for (int k = 0; k < num_tables; ++k) {
                String tag = readStandardString(4);
                rf.skipBytes(4);
                int table_location[] = new int[2];
                table_location[0] = rf.readInt();
                table_location[1] = rf.readInt();
                tables.put(tag, table_location);
            }
            checkCff();
            fontName = getBaseFont();
            fullName = getNames(4); //full name
            familyName = getNames(1); //family name
            allNameEntries = getAllNames();
            if (!justNames) {
                fillTables();
                readGlyphWidths();
                readCMaps();
                readKerning();
                readBbox();
//                GlyphWidths = null;
            }
        }
        finally {
            if (rf != null) {
                rf.close();
                if (!embedded)
                    rf = null;
            }
        }
    }
222471
private Map<File, File> getDataMap(int vcode, String[] dataDirs, String snapshotName) {
		Map<File, File> map = new HashMap<File, File>();

		Map<File, File[]> src = vcode==0 ? getDataMap_1_0(dataDirs) : getDataMap_1_1(dataDirs);
		for (File ks : src.keySet()) {
			File[] sn = src.get(ks);

			if (sn != null) {
				for (int i = 0; i < sn.length; i++) {
					if (snapshotName.equals(sn[i].getName())) {
						map.put(ks, sn[i]);
						break;
					}
				}
			}
		}

		return map;
	}
360494
private void loadHeader(RandomAccessFile fc) throws IOException {
		dbstats.clear();

		ByteBuffer buf = ByteBuffer.allocateDirect(HEADER_SIZE);

		long n = 0;
		int read;
		do {
			read = fc.getChannel().read(buf);
			n += read;
		} while (read > 0); // fill the buffer

		assert n == HEADER_SIZE;

		Check.state(n == HEADER_SIZE, "The file doesn't have a valid OhmDB format!");

		dbstats.put(DbStats.HEADER_SIZE, n);

		buf.rewind();

		String mark = readStr(buf, MARK.length());
		Check.state(mark.equals(MARK), "The file doesn't have a valid OhmDB format (incorrect mark)!");

		int fileVersion = buf.getInt();
		Check.state(fileVersion == 1, "Unsupported OhmDB file format version: %s", fileVersion);

		dbstats.put(DbStats.FILE_VERSION, fileVersion);

		long oddTx1 = buf.getLong();
		long oddTx2 = buf.getLong();
		long oddTx3 = buf.getLong();

		long evenTx1 = buf.getLong();
		long evenTx2 = buf.getLong();
		long evenTx3 = buf.getLong();

		long latestTx = 0;
		if (oddTx1 == oddTx2 && oddTx2 == oddTx3 && oddTx1 > latestTx) {
			latestTx = oddTx1;
			firstTxCounter = false;
		}
		if (evenTx1 == evenTx2 && evenTx2 == evenTx3 && evenTx1 > latestTx) {
			latestTx = evenTx1;
			firstTxCounter = true;
		}

		Long[] commited = { oddTx1, oddTx2, oddTx3, evenTx1, evenTx2, evenTx3 };
		dbstats.put(DbStats.COMMITED, Arrays.asList(commited));

		dbstats.put(DbStats.LATEST_TRANSACTION, latestTx);
		txCounter.set(latestTx);

		// ...

		dbstats.put(DbStats.BLOCK_SIZE, BLOCK_SIZE);

		buf.position(HEADER_SIZE - MARK2.length());
		String mark2 = readStr(buf, MARK2.length());
		Check.state(mark2.equals(MARK2), "The file doesn't have a valid OhmDB format (incorrect mark2)!");
	}