This is unique for top-level jobs, and will be extended with subclass
implementations to trace partitions and split-flows a bit differently.
165896
@Override
    @Trivial
    protected String getExecutionLogMessage(MessageType msgType) {

        LogHelper lh = new LogHelper();
        StringBuilder buf = lh.getBeginningPart(msgType);

        buf.append("invoking execution for a job");
        buf.append("\n JobInstance id = ");
        buf.append(getTopLevelNameInstanceExecutionInfo().getInstanceId());
        buf.append("\n JobExecution id = ");
        buf.append(getTopLevelNameInstanceExecutionInfo().getExecutionId());
        buf.append("\n Job Name = ");
        buf.append(getTopLevelNameInstanceExecutionInfo().getJobName());

        if (jobParameters != null) {
            buf.append("\n Job Parameters = ");
            buf.append(jobParameters);
        }

        if (!msgType.equals(MessageType.STARTED)) {
            buf.append("\n Job Batch Status = " + getBatchStatus());
            buf.append(", Job Exit Status = " + getExitStatus());
        }
        buf.append("\n" + LogHelper.dashes);
        return buf.toString();
    }
80299
private void collectTypeHierarchyInfo(AtlasTypeRegistry              typeRegistry,
                                          Set<String>                    allSuperTypeNames,
                                          Map<String, AtlasAttribute>    allAttributes,
                                          List<String>                   visitedTypes) throws AtlasBaseException {
        if (visitedTypes.contains(entityDef.getName())) {
            throw new AtlasBaseException(AtlasErrorCode.CIRCULAR_REFERENCE, entityDef.getName(),
                                         visitedTypes.toString());
        }

        if (CollectionUtils.isNotEmpty(entityDef.getSuperTypes())) {
            visitedTypes.add(entityDef.getName());
            for (String superTypeName : entityDef.getSuperTypes()) {
                AtlasEntityType superType = typeRegistry.getEntityTypeByName(superTypeName);

                if (superType != null) {
                    superType.collectTypeHierarchyInfo(typeRegistry, allSuperTypeNames, allAttributes, visitedTypes);
                }
            }
            visitedTypes.remove(entityDef.getName());
            allSuperTypeNames.addAll(entityDef.getSuperTypes());
        }

        if (CollectionUtils.isNotEmpty(entityDef.getAttributeDefs())) {
            for (AtlasAttributeDef attributeDef : entityDef.getAttributeDefs()) {

                AtlasType type = typeRegistry.getType(attributeDef.getTypeName());
                allAttributes.put(attributeDef.getName(), new AtlasAttribute(this, attributeDef, type));
            }
        }
    }
33804
function(SuperC, args) {
		args = this._breakdown_subs(args);
		return this._extendclass(
			SuperC,
			args.protos,
			args.recurring,
			args.statics,
			args.name
		);
	}
434891
private void addDirectSubTypes(XClass type, ArrayList subTypes)
    {
        if (type.isInterface())
        {
            if (type.getExtendingInterfaces() != null)
            {
                subTypes.addAll(type.getExtendingInterfaces());
            }
            // we have to traverse the implementing classes as these array contains all classes that
            // implement the interface, not only those who have an "implement" declaration
            // note that for whatever reason the declared interfaces are not exported via the XClass interface
            // so we have to get them via the underlying class which is hopefully a subclass of AbstractClass
            if (type.getImplementingClasses() != null)
            {
                Collection declaredInterfaces = null;
                XClass     subType;

                for (Iterator it = type.getImplementingClasses().iterator(); it.hasNext(); )
                {
                    subType = (XClass)it.next();
                    if (subType instanceof AbstractClass)
                    {
                        declaredInterfaces = ((AbstractClass)subType).getDeclaredInterfaces();
                        if ((declaredInterfaces != null) && declaredInterfaces.contains(type))
                        {
                            subTypes.add(subType);
                        }
                    }
                    else
                    {
                        // Otherwise we have to live with the bug
                        subTypes.add(subType);
                    }
                }
            }
        }
        else
        {
            subTypes.addAll(type.getDirectSubclasses());
        }
    }
189260
private void restartBrokenTransfers(CacheTopology cacheTopology, IntSet addedSegments) {
      Set<Address> members = new HashSet<>(cacheTopology.getReadConsistentHash().getMembers());
      synchronized (transferMapsLock) {
         for (Iterator<Map.Entry<Address, List<InboundTransferTask>>> it = transfersBySource.entrySet().iterator(); it.hasNext(); ) {
            Map.Entry<Address, List<InboundTransferTask>> entry = it.next();
            Address source = entry.getKey();
            if (!members.contains(source)) {
               if (trace) {
                  log.tracef("Removing inbound transfers from source %s for cache %s", source, cacheName);
               }
               List<InboundTransferTask> inboundTransfers = entry.getValue();
               it.remove();
               for (InboundTransferTask inboundTransfer : inboundTransfers) {
                  // these segments will be restarted if they are still in new write CH
                  if (trace) {
                     log.tracef("Removing inbound transfers from node %s for segments %s", source, inboundTransfer.getSegments());
                  }
                  IntSet unfinishedSegments = inboundTransfer.getUnfinishedSegments();
                  inboundTransfer.cancel();
                  addedSegments.addAll(unfinishedSegments);
                  transfersBySegment.keySet().removeAll(unfinishedSegments);
               }
            }
         }

         // exclude those that are already in progress from a valid source
         addedSegments.removeAll(transfersBySegment.keySet());
      }
   }
90436
@Override
    public Pair<Integer, Integer> pPredecessor(int partition, int token) {
        Integer partForToken = m_tokensMap.get().get(token);
        if (partForToken != null && partForToken == partition) {
            Map.Entry<Integer, Integer> predecessor = m_tokensMap.get().headMap(token).lastEntry();

            if (predecessor == null) {
                predecessor = m_tokensMap.get().lastEntry();
            }

            if (predecessor.getKey() != token) {
                return Pair.of(predecessor.getKey(), predecessor.getValue());
            } else {
                // given token is the only one on the ring, umpossible
                throw new RuntimeException("There is only one token on the hash ring");
            }
        } else {
            // given token doesn't map to partition
            throw new IllegalArgumentException("The given token " + token +
                                                   " does not map to partition " + partition);
        }
    }
405373
static PartitionInfo fromEdge(ExecutionEdge executionEdge) {
		final InputChannelDeploymentDescriptor inputChannelDeploymentDescriptor = InputChannelDeploymentDescriptor.fromEdge(executionEdge);

		Preconditions.checkState(
			!inputChannelDeploymentDescriptor.getConsumedPartitionLocation().isUnknown(),
			"PartitionInfo contains an unknown partition location.");

		return new PartitionInfo(
			executionEdge.getSource().getIntermediateResult().getId(),
			inputChannelDeploymentDescriptor);
	}
663545
private void getMoreSplits(JobConf job, Collection<LocatedFileStatus> stats,
                             long maxSize, long minSizeNode,
                             long minSizeRack, long maxNumBlocksPerSplit,
                             List<CombineFileSplit> splits)
    throws IOException {

    // all blocks for all the files in input set
    OneFileInfo[] files;

    // mapping from a rack name to the list of blocks it has
    HashMap<String, List<OneBlockInfo>> rackToBlocks =
                              new HashMap<String, List<OneBlockInfo>>();

    // mapping from a block to the nodes on which it has replicas
    HashMap<OneBlockInfo, String[]> blockToNodes =
                              new HashMap<OneBlockInfo, String[]>();

    // mapping from a node to the list of blocks that it contains
    HashMap<String, List<OneBlockInfo>> nodeToBlocks =
                              new HashMap<String, List<OneBlockInfo>>();

    if (stats.isEmpty()) {
      return;
    }
    files = new OneFileInfo[stats.size()];

    // populate all the blocks for all files
    long totLength = 0;
    int fileIndex = 0;
    for (LocatedFileStatus oneStatus : stats) {
      files[fileIndex] = new OneFileInfo(oneStatus, job,
          isSplitable(FileSystem.get(job), oneStatus.getPath()),
          rackToBlocks, blockToNodes, nodeToBlocks, rackToNodes, maxSize);
      totLength += files[fileIndex].getLength();
      fileIndex++;
    }

    // Sort the blocks on each node from biggest to smallest by size to
    // encourage more node-local single block splits
    sortBlocksBySize(nodeToBlocks);

    ArrayList<OneBlockInfo> validBlocks = new ArrayList<OneBlockInfo>();
    Set<String> nodes = new HashSet<String>();
    long curSplitSize = 0;

    // process all nodes and create splits that are local
    // to a node.
    for (Iterator<Map.Entry<String,
         List<OneBlockInfo>>> iter = nodeToBlocks.entrySet().iterator();
         iter.hasNext();) {

      Map.Entry<String, List<OneBlockInfo>> one = iter.next();
      nodes.add(one.getKey());
      List<OneBlockInfo> blocksInNode = one.getValue();

      // for each block, copy it into validBlocks. Delete it from
      // blockToNodes so that the same block does not appear in
      // two different splits.
      for (OneBlockInfo oneblock : blocksInNode) {
        if (blockToNodes.containsKey(oneblock)) {
          validBlocks.add(oneblock);
          blockToNodes.remove(oneblock);
          curSplitSize += oneblock.length;

          // if the accumulated split size exceeds the maximum, then
          // create this split.
          if ((maxSize != 0 && curSplitSize >= maxSize) ||
              (maxNumBlocksPerSplit > 0 && validBlocks.size() >= maxNumBlocksPerSplit)) {
            // create an input split and add it to the splits array
            // if only one block, add all the node replicas
            if (validBlocks.size() == 1) {
              Set<String> blockLocalNodes =
                new HashSet<String>(Arrays.asList(validBlocks.get(0).hosts));
              addCreatedSplit(job, splits, blockLocalNodes, validBlocks);
              addStatsForSplitType(SplitType.SINGLE_BLOCK_LOCAL, curSplitSize,
                                   blockLocalNodes.size(), validBlocks.size());
            } else {
              addCreatedSplit(job, splits, nodes, validBlocks);
              addStatsForSplitType(SplitType.NODE_LOCAL, curSplitSize,
                                   nodes.size(), validBlocks.size());
            }
            curSplitSize = 0;
            validBlocks.clear();
          }
        }
      }
      // if there were any blocks left over and their combined size is
      // larger than minSplitNode, then combine them into one split.
      // Otherwise add them back to the unprocessed pool. It is likely
      // that they will be combined with other blocks from the same rack later on.
      if (minSizeNode != 0 && curSplitSize >= minSizeNode) {
        // create an input split and add it to the splits array
        addCreatedSplit(job, splits, nodes, validBlocks);
        addStatsForSplitType(SplitType.NODE_LOCAL_LEFTOVER, curSplitSize,
                             nodes.size(), validBlocks.size());
      } else {
        for (OneBlockInfo oneblock : validBlocks) {
          blockToNodes.put(oneblock, oneblock.hosts);
        }
      }
      validBlocks.clear();
      nodes.clear();
      curSplitSize = 0;
    }

    // if blocks in a rack are below the specified minimum size, then keep them
    // in 'overflow'. After the processing of all racks is complete, these overflow
    // blocks will be combined into splits.
    ArrayList<OneBlockInfo> overflowBlocks = new ArrayList<OneBlockInfo>();
    Set<String> racks = new HashSet<String>();

    // Process all racks over and over again until there is no more work to do.
    boolean noRacksMadeSplit = false;
    while (blockToNodes.size() > 0) {

      // Create one split for this rack before moving over to the next rack.
      // Come back to this rack after creating a single split for each of the
      // remaining racks.
      // Process one rack location at a time, Combine all possible blocks that
      // reside on this rack as one split. (constrained by minimum and maximum
      // split size).

      // Iterate over all racks.  Add to the overflow blocks only if at least
      // one pass over all the racks was completed without adding any splits
      long splitsAddedOnAllRacks = 0;
      for (Iterator<Map.Entry<String, List<OneBlockInfo>>> iter =
           rackToBlocks.entrySet().iterator(); iter.hasNext();) {

        Map.Entry<String, List<OneBlockInfo>> one = iter.next();
        racks.add(one.getKey());
        List<OneBlockInfo> blocks = one.getValue();

        // for each block, copy it into validBlocks. Delete it from
        // blockToNodes so that the same block does not appear in
        // two different splits.
        boolean createdSplit = false;
        for (OneBlockInfo oneblock : blocks) {
          if (blockToNodes.containsKey(oneblock)) {
            validBlocks.add(oneblock);
            blockToNodes.remove(oneblock);
            curSplitSize += oneblock.length;

            // if the accumulated split size exceeds the maximum, then
            // create this split.
            if ((maxSize != 0 && curSplitSize >= maxSize) ||
                (maxNumBlocksPerSplit > 0 && validBlocks.size() >= maxNumBlocksPerSplit)) {
              // create an input split and add it to the splits array
              addCreatedSplit(job, splits, getHosts(racks), validBlocks);
              addStatsForSplitType(SplitType.RACK_LOCAL, curSplitSize,
                                   getHosts(racks).size(), validBlocks.size());
              createdSplit = true;
              ++splitsAddedOnAllRacks;
              break;
            }
          }
        }

        // if we created a split, then just go to the next rack
        if (createdSplit) {
          curSplitSize = 0;
          validBlocks.clear();
          racks.clear();
          continue;
        }

        if (!validBlocks.isEmpty()) {
          if (minSizeRack != 0 && curSplitSize >= minSizeRack) {
            // if there is a mimimum size specified, then create a single split
            // otherwise, store these blocks into overflow data structure
            addCreatedSplit(job, splits, getHosts(racks), validBlocks);
            addStatsForSplitType(SplitType.RACK_LOCAL_LEFTOVER, curSplitSize,
                                 getHosts(racks).size(), validBlocks.size());
            ++splitsAddedOnAllRacks;
          } else if (!noRacksMadeSplit) {
            // Add the blocks back if a pass on all rack found at least one
            // split or this is the first pass
            for (OneBlockInfo oneblock : validBlocks) {
              blockToNodes.put(oneblock, oneblock.hosts);
            }
          } else {
            // There were a few blocks in this rack that remained to be processed.
            // Keep them in 'overflow' block list. These will be combined later.
            overflowBlocks.addAll(validBlocks);
          }
        }
        curSplitSize = 0;
        validBlocks.clear();
        racks.clear();
      }

      if (splitsAddedOnAllRacks == 0) {
        noRacksMadeSplit = true;
      }
    }

    assert blockToNodes.isEmpty();
    assert curSplitSize == 0;
    assert validBlocks.isEmpty();
    assert racks.isEmpty();

    // Process all overflow blocks
    for (OneBlockInfo oneblock : overflowBlocks) {
      validBlocks.add(oneblock);
      curSplitSize += oneblock.length;

      // This might cause an exiting rack location to be re-added,
      // but it should be OK because racks is a Set.
      for (int i = 0; i < oneblock.racks.length; i++) {
        racks.add(oneblock.racks[i]);
      }

      // if the accumulated split size exceeds the maximum, then
      // create this split.
      if ((maxSize != 0 && curSplitSize >= maxSize) ||
          (maxNumBlocksPerSplit > 0 && validBlocks.size() >= maxNumBlocksPerSplit)) {
        // create an input split and add it to the splits array
        addCreatedSplit(job, splits, getHosts(racks), validBlocks);
        addStatsForSplitType(SplitType.OVERFLOW, curSplitSize,
                             getHosts(racks).size(), validBlocks.size());
        curSplitSize = 0;
        validBlocks.clear();
        racks.clear();
      }
    }

    // Process any remaining blocks, if any.
    if (!validBlocks.isEmpty()) {
      addCreatedSplit(job, splits, getHosts(racks), validBlocks);
      addStatsForSplitType(SplitType.OVERFLOW_LEFTOVER, curSplitSize,
                           getHosts(racks).size(), validBlocks.size());
    }
  }
715251
private Set<ClassDescriptor> computeKnownSubtypes(ClassDescriptor classDescriptor) throws ClassNotFoundException {
        LinkedList<ClassVertex> workList = new LinkedList<>();

        ClassVertex startVertex = resolveClassVertex(classDescriptor);
        workList.addLast(startVertex);

        Set<ClassDescriptor> result = new HashSet<>();

        while (!workList.isEmpty()) {
            ClassVertex current = workList.removeFirst();

            if (result.contains(current.getClassDescriptor())) {
                // Already added this class
                continue;
            }

            // Add class to the result
            result.add(current.getClassDescriptor());

            // Add all known subtype vertices to the work list
            Iterator<InheritanceEdge> i = graph.incomingEdgeIterator(current);
            while (i.hasNext()) {
                InheritanceEdge edge = i.next();
                workList.addLast(edge.getSource());
            }
        }

        return new HashSet<>(result);
    }
253757
function _gpfClassSuperCreate (superMember) {
    var superMethod;
    if (typeof superMember === "function") {
        superMethod = superMember;
    } else {
        superMethod = _gpfClassNoSuper;
    }
    return _gpfClassSuperCreateWithSameSignature(superMethod);
}
1043879
private void addStatsForSplitType(
      SplitType splitType, long splitSize, long hostCount, long blockCount) {
    SplitTypeStats splitTypeStats = splitTypeStatsMap.get(splitType);
    if (splitTypeStats == null) {
      splitTypeStats = new SplitTypeStats();
      splitTypeStatsMap.put(splitType, splitTypeStats);
    }
    splitTypeStats.addSplit(splitSize, hostCount, blockCount);

    // Add all splits to the ALL split type
    if (splitType != SplitType.ALL) {
      addStatsForSplitType(SplitType.ALL, splitSize, hostCount, blockCount);
    }
  }
70702
private void processNewOperation(StorageOperation operation) {
        if (operation instanceof MergeSegmentOperation) {
            this.operations.add(operation);
            this.mergeTransactionCount.incrementAndGet();
        } else if (operation instanceof StreamSegmentSealOperation) {
            this.operations.add(operation);
            this.hasSealPending.set(true);
        } else if (operation instanceof StreamSegmentTruncateOperation) {
            this.operations.add(operation);
            this.truncateCount.incrementAndGet();
        } else if (operation instanceof CachedStreamSegmentAppendOperation) {
            // Aggregate the Append Operation.
            AggregatedAppendOperation aggregatedAppend = getOrCreateAggregatedAppend(
                    operation.getStreamSegmentOffset(), operation.getSequenceNumber());
            aggregateAppendOperation((CachedStreamSegmentAppendOperation) operation, aggregatedAppend);
        }
    }
784729
private <K> DataStream<T> partitionCustom(Partitioner<K> partitioner, Keys<T> keys) {
		KeySelector<T, K> keySelector = KeySelectorUtil.getSelectorForOneKey(keys, partitioner, getType(), getExecutionConfig());

		return setConnectionType(
				new CustomPartitionerWrapper<>(
						clean(partitioner),
						clean(keySelector)));
	}
1126936
@Override
	void requestSubpartition(int subpartitionIndex) throws IOException, InterruptedException {

		boolean retriggerRequest = false;

		// The lock is required to request only once in the presence of retriggered requests.
		synchronized (requestLock) {
			checkState(!isReleased, "LocalInputChannel has been released already");

			if (subpartitionView == null) {
				LOG.debug("{}: Requesting LOCAL subpartition {} of partition {}.",
					this, subpartitionIndex, partitionId);

				try {
					ResultSubpartitionView subpartitionView = partitionManager.createSubpartitionView(
						partitionId, subpartitionIndex, this);

					if (subpartitionView == null) {
						throw new IOException("Error requesting subpartition.");
					}

					// make the subpartition view visible
					this.subpartitionView = subpartitionView;

					// check if the channel was released in the meantime
					if (isReleased) {
						subpartitionView.releaseAllResources();
						this.subpartitionView = null;
					}
				} catch (PartitionNotFoundException notFound) {
					if (increaseBackoff()) {
						retriggerRequest = true;
					} else {
						throw notFound;
					}
				}
			}
		}

		// Do this outside of the lock scope as this might lead to a
		// deadlock with a concurrent release of the channel via the
		// input gate.
		if (retriggerRequest) {
			inputGate.retriggerPartitionRequest(partitionId.getPartitionId());
		}
	}
732977
public static ArrayList<Trajectory> splitTrackInSubTracks(Trajectory t, int windowWidth, boolean overlapping){
		int increment = 1;
		if(overlapping==false){
			increment=windowWidth;
		}
		ArrayList<Trajectory> subTrajectories = new ArrayList<Trajectory>();
		boolean trackEndReached = false;
		
		for(int i = 0; i < t.size(); i=i+increment)
		{
			int upperBound = i+(windowWidth);
			if(upperBound>t.size()){
				upperBound=t.size();
				trackEndReached=true;
			}
			Trajectory help = new Trajectory(2, i);
			for(int j = i; j < upperBound; j++){
				help.add(t.get(j));
			}
			subTrajectories.add(help);
			if(trackEndReached){
				i=t.size();
			}
		}
		return subTrajectories;
	}
136603
private void processPredecessors(Gantt gantt)
   {
      for (Gantt.Tasks.Task ganttTask : gantt.getTasks().getTask())
      {
         String predecessors = ganttTask.getP();
         if (predecessors != null && !predecessors.isEmpty())
         {
            String wbs = ganttTask.getID();
            Task task = m_taskMap.get(wbs);
            for (String predecessor : predecessors.split(";"))
            {
               Task predecessorTask = m_projectFile.getTaskByID(Integer.valueOf(predecessor));
               task.addPredecessor(predecessorTask, RelationType.FINISH_START, ganttTask.getL());
            }
         }
      }
   }
864667
function _gpfGenSuperMember (superMethod, method) {
    return function GpfSuperableMethod () {
        var previousSuper = this._super,
            result;
        // Add a new ._super() method pointing to the base class member
        this._super = superMethod;
        try {
            // Execute the method
            result = method.apply(this, arguments);
        } finally {
            // Remove it after execution
            if (undefined === previousSuper) {
                delete this._super;
            } else {
                this._super = previousSuper;
            }
        }
        return result;
    };
}