using AngleSharp.Html.Parser; using Extreme.Net; using Newtonsoft.Json.Linq; using RuriLib.LS; using System; using System.Collections.Generic; using System.Text.RegularExpressions; using System.Windows.Media; namespace RuriLib { /// /// The allowed parsing algorithms. /// public enum ParseType { /// Algorithm that parses text between two strings. LR, /// Algorithm that parses a given attribute from an HTML element identified by a CSS Selector. CSS, /// Algorithm that parses values inside a json object. JSON, /// Algorithm that parses a given attribute from an HTML element identified by xpath. XPATH, /// Algorithm that extracts the text inside matched regex groups. REGEX } /// /// A block that parses data from a string. /// public class BlockParse : BlockBase { private string parseTarget = ""; /// The string to parse data from. public string ParseTarget { get { return parseTarget; } set { parseTarget = value; OnPropertyChanged(); } } private string variableName = ""; /// The name of the output variable where the parsed text will be stored. public string VariableName { get { return variableName; } set { variableName = value; OnPropertyChanged(); } } private bool isCapture = false; /// Whether the output variable should be marked as Capture. public bool IsCapture { get { return isCapture; } set { isCapture = value; OnPropertyChanged(); } } private string prefix = ""; /// The string to add to the beginning of the parsed data. public string Prefix { get { return prefix; } set { prefix = value; OnPropertyChanged(); } } private string suffix = ""; /// The string to add to the end of the parsed data. public string Suffix { get { return suffix; } set { suffix = value; OnPropertyChanged(); } } private bool recursive = false; /// Whether to parse multiple values that match the criteria or just the first one. public bool Recursive { get { return recursive; } set { recursive = value; OnPropertyChanged(); } } private bool encodeOutput = false; /// Whether to URL encode the parsed text. public bool EncodeOutput { get { return encodeOutput; } set { encodeOutput = value; OnPropertyChanged(); } } private bool createEmpty = true; /// Whether to create the variable with an empty value if the parsing was not successful. public bool CreateEmpty { get { return createEmpty; } set { createEmpty = value; OnPropertyChanged(); } } private ParseType type = ParseType.LR; /// The parsing algorithm being used. public ParseType Type { get { return type; } set { type = value; OnPropertyChanged(); } } #region LR private string leftString = ""; /// The string to the immediate left of the text to parse. An empty string means the start of the input. public string LeftString { get { return leftString; } set { leftString = value; OnPropertyChanged(); } } private string rightString = ""; /// The string to the immediate right of the text to parse. An empty string means the end of the input. public string RightString { get { return rightString; } set { rightString = value; OnPropertyChanged(); } } private bool useRegexLR = false; /// Whether to use a regex pattern to match a text between two strings instead of the standard algorithm. public bool UseRegexLR { get { return useRegexLR; } set { useRegexLR = value; OnPropertyChanged(); } } #endregion #region CSS private string cssSelector = ""; /// The CSS selector that addresses the desired element in the HTML page. public string CssSelector { get { return cssSelector; } set { cssSelector = value; OnPropertyChanged(); } } private string attributeName = ""; /// The name of the attribute from which to parse the value. public string AttributeName { get { return attributeName; } set { attributeName = value; OnPropertyChanged(); } } private int cssElementIndex = 0; /// The index of the desired element when the selector matches multiple elements. public int CssElementIndex { get { return cssElementIndex; } set { cssElementIndex = value; OnPropertyChanged(); } } #endregion #region JSON private string jsonField = ""; /// The name of the json field for which we want to retrieve the value. public string JsonField { get { return jsonField; } set { jsonField = value; OnPropertyChanged(); } } private bool jTokenParsing = false; /// Whether to parse the json object using jtoken paths. public bool JTokenParsing { get { return jTokenParsing; } set { jTokenParsing = value; OnPropertyChanged(); } } #endregion #region REGEX private string regexString = ""; /// The regex pattern that matches parts of the text inside groups. public string RegexString { get { return regexString; } set { regexString = value; OnPropertyChanged(); } } private string regexOutput = ""; /// The way the content of the matched groups should be formatted. [0] will be replaced with the full match, [1] with the first group etc. public string RegexOutput { get { return regexOutput; } set { regexOutput = value; OnPropertyChanged(); } } #endregion /// /// Creates a Parse block. /// public BlockParse() { Label = "PARSE"; } /// public override BlockBase FromLS(string line) { // Trim the line var input = line.Trim(); // Parse the label if (input.StartsWith("#")) Label = LineParser.ParseLabel(ref input); ParseTarget = LineParser.ParseLiteral(ref input, "TARGET"); Type = (ParseType)LineParser.ParseEnum(ref input, "TYPE", typeof(ParseType)); switch (Type) { case ParseType.LR: // PARSE "" LR "L" "R" RECURSIVE? -> VAR/CAP "ABC" LeftString = LineParser.ParseLiteral(ref input, "LEFT STRING"); RightString = LineParser.ParseLiteral(ref input, "RIGHT STRING"); while (LineParser.Lookahead(ref input) == TokenType.Boolean) LineParser.SetBool(ref input, this); break; case ParseType.CSS: // PARSE "" CSS "Selector" "Attribute" Index RECURSIVE? -> CssSelector = LineParser.ParseLiteral(ref input, "SELECTOR"); AttributeName = LineParser.ParseLiteral(ref input, "ATTRIBUTE"); if (LineParser.Lookahead(ref input) == TokenType.Boolean) LineParser.SetBool(ref input, this); else if (LineParser.Lookahead(ref input) == TokenType.Integer) CssElementIndex = LineParser.ParseInt(ref input, "INDEX"); while (LineParser.Lookahead(ref input) == TokenType.Boolean) LineParser.SetBool(ref input, this); break; case ParseType.JSON: // PARSE "" JSON "Field" -> JsonField = LineParser.ParseLiteral(ref input, "FIELD"); while (LineParser.Lookahead(ref input) == TokenType.Boolean) LineParser.SetBool(ref input, this); break; case ParseType.REGEX: // PARSE "" REGEX "Pattern" "Output" RECURSIVE? -> RegexString = LineParser.ParseLiteral(ref input, "PATTERN"); RegexOutput = LineParser.ParseLiteral(ref input, "OUTPUT"); while (LineParser.Lookahead(ref input) == TokenType.Boolean) LineParser.SetBool(ref input, this); break; } // Parse the arrow LineParser.ParseToken(ref input, TokenType.Arrow, true); // Parse the VAR / CAP try { var varType = LineParser.ParseToken(ref input, TokenType.Parameter, true); if (varType.ToUpper() == "VAR" || varType.ToUpper() == "CAP") IsCapture = varType.ToUpper() == "CAP"; } catch { throw new ArgumentException("Invalid or missing variable type"); } // Parse the variable/capture name try { VariableName = LineParser.ParseLiteral(ref input, "NAME"); } catch { throw new ArgumentException("Variable name not specified"); } // Parse the prefix and suffix try { Prefix = LineParser.ParseLiteral(ref input, "PREFIX"); Suffix = LineParser.ParseLiteral(ref input, "SUFFIX"); } catch { } return this; } /// public override string ToLS(bool indent = true) { var writer = new BlockWriter(GetType(), indent, Disabled); writer .Label(Label) .Token("PARSE") .Literal(ParseTarget) .Token(Type); switch (Type) { case ParseType.LR: writer .Literal(LeftString) .Literal(RightString) .Boolean(Recursive, "Recursive") .Boolean(EncodeOutput, "EncodeOutput") .Boolean(CreateEmpty, "CreateEmpty") .Boolean(UseRegexLR, "UseRegexLR"); break; case ParseType.CSS: writer .Literal(CssSelector) .Literal(AttributeName); if (Recursive) writer.Boolean(Recursive, "Recursive"); else writer.Integer(CssElementIndex, "CssElementIndex"); writer .Boolean(EncodeOutput, "EncodeOutput") .Boolean(CreateEmpty, "CreateEmpty"); break; case ParseType.JSON: writer .Literal(JsonField) .Boolean(JTokenParsing, "JTokenParsing") .Boolean(Recursive, "Recursive") .Boolean(EncodeOutput, "EncodeOutput") .Boolean(CreateEmpty, "CreateEmpty"); break; case ParseType.REGEX: writer .Literal(RegexString) .Literal(RegexOutput) .Boolean(Recursive, "Recursive") .Boolean(EncodeOutput, "EncodeOutput") .Boolean(CreateEmpty, "CreateEmpty"); break; } writer .Arrow() .Token(IsCapture ? "CAP" : "VAR") .Literal(VariableName); if (!writer.CheckDefault(Prefix, "Prefix") || !writer.CheckDefault(Suffix, "Suffix")) writer.Literal(Prefix).Literal(Suffix); return writer.ToString(); } /// public override void Process(BotData data) { base.Process(data); InsertVariables(data, isCapture, recursive, Parse(data), variableName, prefix, suffix, encodeOutput, createEmpty); } private List Parse(BotData data) { var original = ReplaceValues(parseTarget, data); var partial = original; var list = new List(); // Parse the value switch (Type) { case ParseType.LR: var ls = ReplaceValues(leftString, data); var rs = ReplaceValues(rightString, data); var pFrom = 0; var pTo = 0; // No L and R = return full input if (ls == "" && rs == "") { list.Add(original); break; } // L or R not present and not empty else if ( ((!partial.Contains(ls) && ls != "") || (!partial.Contains(rs) && rs != ""))) { break; } // Instead of the mess below, we could simply use Extreme.NET's Substring extensions // return original.Substrings(ls, rs); // Recursive // return original.Substring(ls, rs); // Not recursive if (recursive) { if (useRegexLR) { try { var pattern = BuildLRPattern(ls, rs); MatchCollection mc = Regex.Matches(partial, pattern); foreach (Match m in mc) list.Add(m.Value); } catch { } } else { try { while ((partial.Contains(ls) || ls == "") && (partial.Contains(rs) || rs == "")) { // Search for left delimiter and Calculate offset pFrom = ls == "" ? 0 : partial.IndexOf(ls) + ls.Length; // Move right of offset partial = partial.Substring(pFrom); // Search for right delimiter and Calculate length to parse pTo = rs == "" ? (partial.Length - 1) : partial.IndexOf(rs); // Parse it var parsed = partial.Substring(0, pTo); list.Add(parsed); // Move right of parsed + right partial = partial.Substring(parsed.Length + rs.Length); } } catch { } } } // Non-recursive else { if (useRegexLR) { var pattern = BuildLRPattern(ls, rs); MatchCollection mc = Regex.Matches(partial, pattern); if (mc.Count > 0) list.Add(mc[0].Value); } else { try { pFrom = ls == "" ? 0 : partial.IndexOf(ls) + ls.Length; partial = partial.Substring(pFrom); pTo = rs == "" ? partial.Length : partial.IndexOf(rs); list.Add(partial.Substring(0, pTo)); } catch { } } } break; case ParseType.CSS: HtmlParser parser = new HtmlParser(); AngleSharp.Html.Dom.IHtmlDocument document = null; try { document = parser.ParseDocument(original); } catch { } try { if (recursive) { foreach(var element in document.QuerySelectorAll(ReplaceValues(cssSelector,data))) { switch (ReplaceValues(attributeName, data)) { case "innerHTML": list.Add(element.InnerHtml); break; case "outerHTML": list.Add(element.OuterHtml); break; default: foreach(var attr in element.Attributes) { if(attr.Name == ReplaceValues(attributeName,data)) { list.Add(attr.Value); break; } } break; } } } else { switch (ReplaceValues(attributeName, data)) { case "innerHTML": list.Add(document.QuerySelectorAll(ReplaceValues(cssSelector, data))[cssElementIndex].InnerHtml); break; case "outerHTML": list.Add(document.QuerySelectorAll(ReplaceValues(cssSelector, data))[cssElementIndex].OuterHtml); break; default: foreach (var attr in document.QuerySelectorAll(ReplaceValues(cssSelector, data))[cssElementIndex].Attributes) { if (attr.Name == ReplaceValues(attributeName, data)) { list.Add(attr.Value); break; } } break; } } } catch { } break; case ParseType.JSON: if (JTokenParsing) { JObject json = JObject.Parse(original); var jsonlist = json.SelectTokens(jsonField, false); foreach (var j in jsonlist) list.Add(j.ToString()); } else { var jsonlist = new List>(); parseJSON("", original, jsonlist); foreach (var j in jsonlist) if (j.Key == ReplaceValues(jsonField, data)) list.Add(j.Value); } break; case ParseType.XPATH: // NOT IMPLEMENTED YET break; case ParseType.REGEX: try { var matches = Regex.Matches(partial, ReplaceValues(regexString, data)); foreach(Match match in matches) { var output = ReplaceValues(regexOutput, data); for (var i = 0; i < match.Groups.Count; i++) output = output.Replace("[" + i + "]", match.Groups[i].Value); list.Add(output); } } catch { } break; } return list; } private string BuildLRPattern(string ls, string rs) { var left = string.IsNullOrEmpty(ls) ? "^" : Regex.Escape(ls); // Empty LEFT = start of the line var right = string.IsNullOrEmpty(rs) ? "$" : Regex.Escape(rs); // Empty RIGHT = end of the line return "(?<=" + left + ").+?(?=" + right + ")"; } private static void parseJSON(string A, string B, List> jsonlist) { jsonlist.Add(new KeyValuePair(A, B)); if (B.StartsWith("[")) { JArray arr = null; try { arr = JArray.Parse(B); } catch { return; } foreach (var i in arr.Children()) parseJSON("", i.ToString(), jsonlist); } if (B.Contains("{")) { JObject obj = null; try { obj = JObject.Parse(B); } catch { return; } foreach (var o in obj) parseJSON(o.Key, o.Value.ToString(), jsonlist); } } private string cleanString(string inputString) { return Regex.Replace(inputString, "
", "").Trim(); } } }