!7

static File mainLog, fTriples, file, file2;

p {
  //if (isMain()) restartWith2GBHeap();
  
  phase1();

  print("PHASE 2 in 5 SECONDS");
  sleepSeconds(5);
  phase2();
}

svoid phase1 {
  // First, rotate.
  
  mainLog = programFile(#1010745, "webs-made.txt");
  fTriples = programFile(#1010745, "triples.gz");
  rotateLogFile(mainLog);
  
  File tripelizedDir = programDir(#1010745, "tripelized");
  
  // Load webs

  new Map<Web, SoftwareMadeWeb> fiMap;
  L<Web> webs = websMadeByProgram(#1010745, fiMap);
  
  // Get all references to global IDs
  new HashSet<S> allReferences;
  for (Web web : webs)
    for (WebNode n : web_nodesAndRelations(web))
      for (S s : web_texts(n))
        allReferences.addAll(aggressivelyCollectPossibleGlobalIDs(s));
        
  // Classify by content & find unreferenced duplicates
  // key = triple + verified
  HashMap<Pair<T3<S>, Bool>, GlobalID> websByContent = new HashMap;
  new CompactHashSet<GlobalID> unreferencedDuplicates;
  for (Web web : webs) {
    Pair<T3<S>, Bool> key = pair(webToTriple(web), web.verified());
    if (websByContent.containsKey(key)) {
      if (!allReferences.contains(web.globalID()))
        unreferencedDuplicates.add(web.globalIDObj());
    } else
      websByContent.put(key, web.globalIDObj());
  }
  print("Have " + n(unreferencedDuplicates, "unreferenced duplicate web") + ", dropping.");
  
  // Drop unreferenced duplicates
  webs = webs_dropWebsContainedInIDSet(webs, unreferencedDuplicates);
  print("Now have " + nWeb(webs) + ".");
    
  // Drop invalid webs
  new HashMap<S> invalidatedBy;
  new Matches m;
  for (Web web : webs)
    if (web.verified())
      for (WebNode n : web_search_dollarX(webFromTriple("$X", "is", "invalid"), web))
        if (web_match("Web *", n, m) && isGlobalID($1))
          invalidatedBy.put($1, web.globalID());
  for (S id : cloneKeys(invalidatedBy)) {
    S invalidator = invalidatedBy.get(id);
    if (invalidatedBy.containsKey(invalidator))
      invalidatedBy.remove(id);
  }
  print("Removing " + n(invalidatedBy, "invalid web"));
  saveTextFile(countTillNewFile(newFile(tripelizedDir, "invalidated"), ".gz"), structure(invalidatedBy));
  webs = webs_dropIDs(webs, keys(invalidatedBy));
  
  int dropped = 0, dropped2 = 0;
  new L<Web> websOut;
  for (Web web : webs) {
  if (web_nodesTooLong(web))
      dropped++;
    else if (!web_tripelizable(web))
      dropped2++;
    else
      websOut.add(web);
  }
  webs = websOut;
  if (dropped != 0)
    print("Dropped " + nWeb(dropped) + " with too long nodes");
  if (dropped2 != 0)
    print("Dropped " + n(dropped2, "non-triple"));
  
  Pair<L<Web>> pair = webs_tripelizable_nonTripelizable(webs);

  // Save triples
  
  file = new File(tripelizedDir, "triples.new.gz");
  PrintWriter out = newPrintWriter(newGZIPOutputStream(file));
  webs_toTripleFile(pair.a, out);
  out.close();

  // Save non-triples
  
  file2 = new File(tripelizedDir, "webs-made.new.gz");
  out = newPrintWriter(newGZIPOutputStream(file2));
  for (Web web : pair.b)
    out.println(quote(struct(fiMap.get(web))));
  out.close();
  
  print("Stored " + nWeb(pair.a) + " in " + f2s(file) + ", " + nWeb(pair.b) + " in " + f2s(file2));
}
 
svoid phase2 {
  print("PHASE 2.");
  
  // Move data to backups
  
  File fBackups = programFile(#1010745, "backups");
  for (File f : earlierPartsOfLogFile(mainLog)) {
    int n = 1;
    File p;
    while ((p = newFile(fBackups, dropSuffix(".gz", f.getName()) + ".part" + n + ".gz")).exists()) ++n;
    renameFile_assertTrue(f, p);
  }
  
  // Backup triples.gz
  
  int n = 1;
  File p;
  while ((p = newFile(fBackups, dropSuffix(".gz", fTriples.getName()) + ".part" + n + ".gz")).exists()) ++n;
  renameFile_assertTrue(fTriples, p);
  
  // Rename main files
  renameFile_assertTrue(file, fTriples);
  renameFile_assertTrue(file2, programFile(#1010745, "webs-made.txt.part1.gz"));
  
  print("DONE.");
}