!752

static S html;
static S keyword;
static new List<S> sentences;
static S source;

p {
  if (args.length != 0) {
    if (isSnippetID(args[0]))
      html = loadSnippet(args[0]);
    else if (args[0].startsWith("http:") || args[0].startsWith("https://"))
      html = loadPage(source = args[0]);
    else {
      keyword = join(" ", args);
      html = loadPage(source = "https://en.wikipedia.org/wiki/" + keyword);
    }
  } else
    html = loadSnippet(source = "#3000030"); // A text about Java
  print();
    
  // tokenize and clean up html
  L<S> tok = htmlcoarsetok(html);
  tok = removeScripts(tok);
  tok = dropTags(tok, "span");
  tok = dropTags(tok, "a");
  tok = dropTags(tok, "b");
  tok = dropTags(tok, "small");
  
  // tok = dropAllTags(tok); // Too much!
  //print(structure(tok));

  for (int i = 0; i < tok.size(); i += 2) {
    S line = tok.get(i).trim();
    if (line.startsWith("<!--")) continue; // HTML comment
    if (line.length() == 0) continue;
    //print("Line: " + quote(line));
    interpretLine(line);
  }
  
  print();
  print(sentences.size() + " sentences found.");
  print();
  
  ntUpload(getProgramID(), "Sentences from " + source, makeText());
  
  if (keyword != null) {
    S shortenedKeyword = keyword.replaceAll(" \\(.*$", "").trim().toLowerCase();
    print("Here are those about " + quote(keyword) + ":");
    print();
    for (S s : sentences) {
      S x = s.toLowerCase();
      if (x.startsWith(shortenedKeyword) || x.startsWith("The " + shortenedKeyword) || x.startsWith("A " + shortenedKeyword) || x.startsWith("An " + shortenedKeyword))
        print(s);
    }
  }
}

static S makeText() {
  ret numberLines(fromLines(sentences));
}

static boolean hasCharacters(S s) {
  for (int i = 0; i < s.length(); i++)
    if (Character.isLetter(s.charAt(i)))
      ret true;
  ret false;
}

static void interpretLine(S s) {
  for (S sentence : getSentences(s)) {
    char first = sentence.charAt(0);
    if (Character.isLowerCase(first) || ",;:=".indexOf(first) >= 0) continue;
    if (!hasCharacters(sentence)) continue;
    //L<S> tok = javaTok(sentence);
    //print(structure(tok));
    sentence = htmldecode(sentence);
    if (!isAndroid())
      print("Sentence found: " + quote(sentence));
    sentences.add(sentence);
  }
}

static L<S> getSentences(S s) {
  L<S> tok = javaTok(s); // To parse quoted things
  fixSpaces(tok);
  new L<S> list;
  int i = 0;
  while (true) {
    int j = i;
    do {
      j = indexOf(tok, ".", j+1);
      if (j < 0) return list;
    } while (j+1 < tok.size()-1 && tok.get(j+1).equals("")); // matches stuff like "9.5"
    
    S sentence = join(tok.subList(i, j+1)).trim();
    if (sentence.length() > 1)
      list.add(sentence);
    i = j+1;
  }
}

// So everything's on one line.
static void fixSpaces(L<S> tok) {
  for (int i = 0; i < l(tok); i += 2)
    tok.set(i, tok.get(i).equals("") ? "" : " ");
}

static L<S> removeScripts(L<S> tok) {
  tok = new ArrayList<S>(tok);
  for (int i = 1; i < tok.size(); )
    if (tagIs(tok.get(i), "script")) {
      int j = i;
      while (j < tok.size() && !tagIs(tok.get(j), "/script"))
        j += 2;
      while (j+1 > i-1) { // Yeah it's inefficient...
        tok.remove(i-1); --j; 
      }
    } else
      i += 2;
  return tok;
}

static L<S> dropTags(L<S> tok, S tag) {
  new L<S> list;
  for (int i = 0; i < tok.size(); i++) {
    S t = tok.get(i);
    if (tagIs(t, tag) || tagIs(t, "/" + tag)) {
      list.set(list.size()-1, list.get(list.size()-1) + tok.get(i+1));
      ++i;
    } else
      list.add(tok.get(i));
  }
  return list;
}

static L<S> dropAllTags(L<S> tok) {
  new L<S> list;
  for (int i = 0; i < tok.size(); i++) {
    S t = tok.get(i);
    if (t.startsWith("<")) {
      list.set(list.size()-1, list.get(list.size()-1) + tok.get(i+1));
      ++i;
    } else
      list.add(tok.get(i));
  }
  return list;
}