using Extreme.Net; using RuriLib.LS; using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.IO; using System.Linq; using System.Text; using System.Windows.Media; namespace RuriLib { /// /// The types of request that can be performed. /// public enum RequestType { /// A standard request with standard content. Standard, /// A request which uses the 'Authentication: Basic' header. BasicAuth, /// A request which contains multipart content (strings and/or files). Multipart } /// /// The available types of multipart contents. /// public enum MultipartContentType { /// A string content. String, /// A file content. File } /// /// The type of data expected inside the HTTP response. /// public enum ResponseType { /// A string response, e.g. an HTML page. String, /// A file response, e.g. an image. File } /// /// A block that can perform HTTP requests. /// public class BlockRequest : BlockBase { #region Variables private string url = "https://google.com"; /// The URL to call, including additional GET query parameters. public string Url { get { return url; } set { url = value; OnPropertyChanged(); } } private RequestType requestType = RequestType.Standard; /// The request type. public RequestType RequestType { get { return requestType; } set { requestType = value; OnPropertyChanged(); } } // Basic Auth private string authUser = ""; /// The username for basic auth requests. public string AuthUser { get { return authUser; } set { authUser = value; OnPropertyChanged(); } } private string authPass = ""; /// The password for basic auth requests. public string AuthPass { get { return authPass; } set { authPass = value; OnPropertyChanged(); } } // Standard private string postData = ""; /// The content of the request, sent after the headers. Use '\n' to input a linebreak. public string PostData { get { return postData; } set { postData = value; OnPropertyChanged(); } } private HttpMethod method = HttpMethod.GET; /// The method of the HTTP request. public HttpMethod Method { get { return method; } set { method = value; OnPropertyChanged(); } } /// The custom headers that are sent in the HTTP request. public Dictionary CustomHeaders { get; set; } = new Dictionary() { { "User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko" }, { "Pragma", "no-cache" }, { "Accept", "*/*" } }; /// The custom cookies that are sent in the HTTP request. public Dictionary CustomCookies { get; set; } = new Dictionary() { }; private string contentType = "application/x-www-form-urlencoded"; /// The type of content the server should expect. public string ContentType { get { return contentType; } set { contentType = value; OnPropertyChanged(); } } private bool autoRedirect = true; /// Whether to perform automatic redirection in the case of 3xx headers. public bool AutoRedirect { get { return autoRedirect; } set { autoRedirect = value; OnPropertyChanged(); } } private bool readResponseSource = true; /// Whether to read the stream of data from the HTTP response. Set to false if only the headers are needed, in order to speed up the process. public bool ReadResponseSource { get { return readResponseSource; } set { readResponseSource = value; OnPropertyChanged(); } } private bool parseQuery = false; /// Whether to parse the GET parameters manually (fixes Extreme.NET issues on some websites). public bool ParseQuery { get { return parseQuery; } set { parseQuery = value; OnPropertyChanged(); } } private bool encodeContent = false; /// Whether to URL encode the content before sending it. public bool EncodeContent { get { return encodeContent; } set { encodeContent = value; OnPropertyChanged(); } } private bool acceptEncoding = true; /// Whether to automatically generate an Accept-Encoding header. public bool AcceptEncoding { get { return acceptEncoding; } set { acceptEncoding = value; OnPropertyChanged(); } } // Multipart private string multipartBoundary = ""; /// The boundary that separates multipart contents. public string MultipartBoundary { get { return multipartBoundary; } set { multipartBoundary = value; OnPropertyChanged(); } } /// The list of contents to send in a multipart request. public List MultipartContents { get; set; } = new List(); private ResponseType responseType = ResponseType.String; /// The type of response expected from the server. public ResponseType ResponseType { get { return responseType; } set { responseType = value; OnPropertyChanged(); } } private string downloadPath = ""; /// The path of the file where a FILE response needs to be stored. public string DownloadPath { get { return downloadPath; } set { downloadPath = value; OnPropertyChanged(); } } #endregion /// /// Creates a Request block. /// public BlockRequest() { Label = "REQUEST"; } /// public override BlockBase FromLS(string line) { // Trim the line var input = line.Trim(); // Parse the label if (input.StartsWith("#")) Label = LineParser.ParseLabel(ref input); Method = (HttpMethod)LineParser.ParseEnum(ref input, "METHOD", typeof(HttpMethod)); Url = LineParser.ParseLiteral(ref input, "URL"); while (LineParser.Lookahead(ref input) == TokenType.Boolean) LineParser.SetBool(ref input, this); CustomHeaders.Clear(); // Remove the default headers while (input != "" && !input.StartsWith("->")) { var parsed = LineParser.ParseToken(ref input, TokenType.Parameter, true).ToUpper(); switch (parsed) { case "MULTIPART": RequestType = RequestType.Multipart; break; case "BASICAUTH": RequestType = RequestType.BasicAuth; break; case "STANDARD": RequestType = RequestType.Standard; break; case "CONTENT": PostData = LineParser.ParseLiteral(ref input, "POST DATA"); break; case "STRINGCONTENT": var stringContentPair = ParseString(LineParser.ParseLiteral(ref input, "STRING CONTENT"), ':', 2); MultipartContents.Add(new MultipartContent() { Type = MultipartContentType.String, Name = stringContentPair[0], Value = stringContentPair[1] }); break; case "FILECONTENT": var fileContentTriplet = ParseString(LineParser.ParseLiteral(ref input, "FILE CONTENT"), ':', 3); MultipartContents.Add(new MultipartContent() { Type = MultipartContentType.File, Name = fileContentTriplet[0], Value = fileContentTriplet[1], ContentType = fileContentTriplet[2] }); break; case "COOKIE": var cookiePair = ParseString(LineParser.ParseLiteral(ref input, "COOKIE VALUE"), ':', 2); CustomCookies[cookiePair[0]] = cookiePair[1]; break; case "HEADER": var headerPair = ParseString(LineParser.ParseLiteral(ref input, "HEADER VALUE"), ':', 2); CustomHeaders[headerPair[0]] = headerPair[1]; break; case "CONTENTTYPE": ContentType = LineParser.ParseLiteral(ref input, "CONTENT TYPE"); break; case "USERNAME": AuthUser = LineParser.ParseLiteral(ref input, "USERNAME"); break; case "PASSWORD": AuthPass = LineParser.ParseLiteral(ref input, "PASSWORD"); break; case "BOUNDARY": MultipartBoundary = LineParser.ParseLiteral(ref input, "BOUNDARY"); break; default: break; } } if (input.StartsWith("->")) { LineParser.EnsureIdentifier(ref input, "->"); var outType = LineParser.ParseToken(ref input, TokenType.Parameter, true); if (outType.ToUpper() == "STRING") ResponseType = ResponseType.String; else if (outType.ToUpper() == "FILE") { ResponseType = ResponseType.File; DownloadPath = LineParser.ParseLiteral(ref input, "DOWNLOAD PATH"); } } return this; } /// /// Parses values from a string. /// /// The string to parse /// The character that separates the elements /// The number of elements to return /// The array of the parsed elements. public static string[] ParseString(string input, char separator, int count) { return input.Split(new[] { separator }, count).Select(s => s.Trim()).ToArray(); } /// public override string ToLS(bool indent = true) { var writer = new BlockWriter(GetType(), indent, Disabled); writer .Label(Label) .Token("REQUEST") .Token(Method) .Literal(Url) .Boolean(AcceptEncoding, "AcceptEncoding") .Boolean(AutoRedirect, "AutoRedirect") .Boolean(ReadResponseSource, "ReadResponseSource") .Boolean(ParseQuery, "ParseQuery") .Boolean(EncodeContent, "EncodeContent") .Token(RequestType, "RequestType") .Indent(); switch (RequestType) { case RequestType.BasicAuth: writer .Token("USERNAME") .Literal(AuthUser) .Token("PASSWORD") .Literal(AuthPass) .Indent(); break; case RequestType.Standard: if (CanContainBody(method)) { writer .Token("CONTENT") .Literal(PostData) .Indent() .Token("CONTENTTYPE") .Literal(ContentType); } break; case RequestType.Multipart: foreach(var c in MultipartContents) { writer .Indent() .Token($"{c.Type.ToString().ToUpper()}CONTENT"); if (c.Type == MultipartContentType.String) { writer.Literal($"{c.Name}: {c.Value}"); } else if (c.Type == MultipartContentType.File) { writer.Literal($"{c.Name}: {c.Value}: {c.ContentType}"); } } if (!writer.CheckDefault(MultipartBoundary, "MultipartBoundary")) { writer .Indent() .Token("BOUNDARY") .Literal(MultipartBoundary); } break; } foreach (var c in CustomCookies) { writer .Indent() .Token("COOKIE") .Literal($"{c.Key}: {c.Value}"); } foreach (var h in CustomHeaders) { writer .Indent() .Token("HEADER") .Literal($"{h.Key}: {h.Value}"); } if (ResponseType == ResponseType.File) { writer .Indent() .Arrow() .Token("FILE") .Literal(DownloadPath); } return writer.ToString(); } /// public override void Process(BotData data) { base.Process(data); #region Request // Set base URL var localUrl = ReplaceValues(url, data); var cType = ReplaceValues(contentType, data); // Create request HttpRequest request = new HttpRequest(); // Setup options var timeout = data.GlobalSettings.General.RequestTimeout * 1000; request.IgnoreProtocolErrors = true; request.AllowAutoRedirect = autoRedirect; request.EnableEncodingContent = acceptEncoding; request.ReadWriteTimeout = timeout; request.ConnectTimeout = timeout; request.KeepAlive = true; request.MaximumAutomaticRedirections = data.ConfigSettings.MaxRedirects; // Check if it has GET parameters if (ParseQuery && localUrl.Contains('?') && localUrl.Contains('=')) { // Remove the query from the base URL localUrl = ReplaceValues(url.Split('?')[0], data); data.Log(new LogEntry($"Calling Base URL: {localUrl}", Colors.MediumTurquoise)); // Parse the GET parameters var getParams = ReplaceValues(url.Split('?')[1], data); var paramList = getParams.Split('&'); // Build the query, first replace variables in them and encode the parameters foreach (var par in paramList) { var split = par.Split('='); // Encode them if needed if (split[0].Contains('%')) split[0] = Uri.EscapeDataString(split[0]); if (split[1].Contains('%')) split[1] = Uri.EscapeDataString(split[1]); // Add them to the query request.AddUrlParam(split[0], split[1]); data.Log(new LogEntry($"Added Query Parameter: {split[0]} = {split[1]}", Colors.MediumTurquoise)); } } else { data.Log(new LogEntry($"Calling URL: {localUrl}", Colors.MediumTurquoise)); } // Set up the Content and Content-Type HttpContent content = null; switch (requestType) { case RequestType.Standard: var pData = string.Join(Environment.NewLine, postData .Split(new string[] { "\\n" }, StringSplitOptions.None) .Select(p => ReplaceValues(p, data))); if (CanContainBody(method)) { if (encodeContent) { // Very dirty but it works var nonce = data.Random.Next(1000000, 9999999); pData = pData.Replace("&", $"{nonce}&{nonce}").Replace("=", $"{nonce}={nonce}"); pData = System.Uri.EscapeDataString(pData).Replace($"{nonce}%26{nonce}", "&").Replace($"{nonce}%3D{nonce}", "="); } content = new StringContent(pData); content.ContentType = cType; data.Log(new LogEntry($"Post Data: {pData}", Colors.MediumTurquoise)); } break; case RequestType.Multipart: var bdry = multipartBoundary != "" ? ReplaceValues(multipartBoundary, data) : GenerateMultipartBoundary(); content = new Extreme.Net.MultipartContent(bdry); var mContent = content as Extreme.Net.MultipartContent; data.Log(new LogEntry($"Content-Type: multipart/form-data; boundary={bdry}", Colors.MediumTurquoise)); data.Log(new LogEntry("Multipart Data:", Colors.MediumTurquoise)); data.Log(new LogEntry(bdry, Colors.MediumTurquoise)); foreach (var c in MultipartContents) { var rValue = ReplaceValues(c.Value, data); var rName = ReplaceValues(c.Name, data); var rContentType = ReplaceValues(c.ContentType, data); if (c.Type == MultipartContentType.String) { mContent.Add(new StringContent(rValue), rName); data.Log(new LogEntry($"Content-Disposition: form-data; name=\"{rName}\"{Environment.NewLine}{Environment.NewLine}{rValue}", Colors.MediumTurquoise)); } else if (c.Type == MultipartContentType.File) { mContent.Add(new FileContent(rValue), rName, rValue, rContentType); data.Log(new LogEntry($"Content-Disposition: form-data; name=\"{rName}\"; filename=\"{rValue}\"{Environment.NewLine}Content-Type: {rContentType}{Environment.NewLine}{Environment.NewLine}[FILE CONTENT OMITTED]", Colors.MediumTurquoise)); } data.Log(new LogEntry(bdry, Colors.MediumTurquoise)); } break; default: break; } // Set proxy if (data.UseProxies) { request.Proxy = data.Proxy.GetClient(); try { request.Proxy.ReadWriteTimeout = timeout; request.Proxy.ConnectTimeout = timeout; request.Proxy.Username = data.Proxy.Username; request.Proxy.Password = data.Proxy.Password; } catch { } } // Set headers data.Log(new LogEntry("Sent Headers:", Colors.DarkTurquoise)); // var fixedNames = Enum.GetNames(typeof(HttpHeader)).Select(n => n.ToLower()); foreach (var header in CustomHeaders) { try { var key = ReplaceValues(header.Key, data); var replacedKey = key.Replace("-", "").ToLower(); // Used to compare with the HttpHeader enum var val = ReplaceValues(header.Value, data); if (replacedKey == "contenttype" && content != null) { continue; } // Disregard additional Content-Type headers if (replacedKey == "acceptencoding" && acceptEncoding) { continue; } // Disregard additional Accept-Encoding headers // else if (fixedNames.Contains(replacedKey)) request.AddHeader((HttpHeader)Enum.Parse(typeof(HttpHeader), replacedKey, true), val); else request.AddHeader(key, val); data.Log(new LogEntry(key + ": " + val, Colors.MediumTurquoise)); } catch { } } // Add the authorization header on a Basic Auth request if (requestType == RequestType.BasicAuth) { var usr = ReplaceValues(authUser, data); var pwd = ReplaceValues(authPass, data); var auth = "Basic " + BlockFunction.Base64Encode(usr + ":" + pwd); request.AddHeader("Authorization", auth); data.Log(new LogEntry("Authorization: " + auth, Colors.MediumTurquoise)); } // Add the content-type header if (CanContainBody(method) && content != null && requestType == RequestType.Standard) { data.Log(new LogEntry("Content-Type: " + cType, Colors.MediumTurquoise)); } // Add new user-defined custom cookies to the bot's cookie jar request.Cookies = new CookieDictionary(); foreach (var cookie in CustomCookies) data.Cookies[ReplaceValues(cookie.Key, data)] = ReplaceValues(cookie.Value, data); // Set cookies from the bot's cookie jar to the request's CookieDictionary data.Log(new LogEntry("Sent Cookies:", Colors.MediumTurquoise)); foreach (var cookie in data.Cookies) { request.Cookies.Add(cookie.Key, cookie.Value); data.Log(new LogEntry(cookie.Key + " : " + cookie.Value, Colors.MediumTurquoise)); } data.LogNewLine(); #endregion #region Response // Create the response HttpResponse response = null; try { // Get response response = request.Raw(method, localUrl, content); // Get address data.Address = response.Address.ToString(); data.Log(new LogEntry("Address: " + data.Address, Colors.Cyan)); // Get code data.ResponseCode = ((int)response.StatusCode).ToString(); data.Log(new LogEntry($"Response code: {data.ResponseCode} ({response.StatusCode})", Colors.Cyan)); // Get headers data.Log(new LogEntry("Received headers:", Colors.DeepPink)); var headerList = new List>(); var receivedHeaders = response.EnumerateHeaders(); data.ResponseHeaders.Clear(); while (receivedHeaders.MoveNext()) { var header = receivedHeaders.Current; data.ResponseHeaders.Add(header.Key, header.Value); data.Log(new LogEntry(header.Key + ": " + header.Value, Colors.LightPink)); } // Get cookies data.Log(new LogEntry("Received cookies:", Colors.Goldenrod)); data.Cookies = response.Cookies; foreach (var cookie in response.Cookies) { data.Log(new LogEntry(cookie.Key + ": " + cookie.Value, Colors.LightGoldenrodYellow)); } // Save the response content switch (responseType) { case ResponseType.String: data.Log(new LogEntry("Response Source:", Colors.Green)); if (readResponseSource) { data.ResponseSource = response.ToString(); data.Log(new LogEntry(data.ResponseSource, Colors.GreenYellow)); } else { data.ResponseSource = ""; data.Log(new LogEntry("[SKIPPED]", Colors.GreenYellow)); } break; case ResponseType.File: var file = ReplaceValues(downloadPath, data); using (var stream = File.Create(file)) { response.ToMemoryStream().CopyTo(stream); } data.Log(new LogEntry("File saved as " + file, Colors.Green)); break; default: break; } } catch (Exception ex) { data.Log(new LogEntry(ex.Message, Colors.White)); if (ex.GetType() == typeof(HttpException)) { data.ResponseCode = ((HttpException)ex).HttpStatusCode.ToString(); data.Log(new LogEntry("Status code: " + data.ResponseCode, Colors.Cyan)); } if (!data.ConfigSettings.IgnoreResponseErrors) throw; } #endregion } #region Custom Cookies, Headers and Multipart Contents /// /// Builds a string containing custom cookies. /// /// One cookie per line, with name and value separated by a colon public string GetCustomCookies() { StringBuilder sb = new StringBuilder(); foreach (var pair in CustomCookies) { sb.Append($"{pair.Key}: {pair.Value}"); if (!pair.Equals(CustomCookies.Last())) sb.Append(Environment.NewLine); } return sb.ToString(); } /// /// Sets custom cookies from an array of lines. /// /// The lines containing the colon-separated name and value of the cookies public void SetCustomCookies(string[] lines) { CustomCookies.Clear(); foreach (var line in lines) { if (line.Contains(':')) { var split = line.Split(new[] { ':' }, 2); CustomCookies[split[0].Trim()] = split[1].Trim(); } } } /// /// Builds a string containing custom headers. /// /// One header per line, with name and value separated by a colon public string GetCustomHeaders() { StringBuilder sb = new StringBuilder(); foreach (var pair in CustomHeaders) { sb.Append($"{pair.Key}: {pair.Value}"); if (!pair.Equals(CustomHeaders.Last())) sb.Append(Environment.NewLine); } return sb.ToString(); } /// /// Sets custom headers from an array of lines. /// /// The lines containing the colon-separated name and value of the headers public void SetCustomHeaders(string[] lines) { CustomHeaders.Clear(); foreach (var line in lines) { if (line.Contains(':')) { var split = line.Split(new[] { ':' }, 2); CustomHeaders[split[0].Trim()] = split[1].Trim(); } } } /// /// Builds a string containing multipart content. /// /// One content per line, with type, name and value separated by a colon public string GetMultipartContents() { StringBuilder sb = new StringBuilder(); foreach (var c in MultipartContents) { sb.Append($"{c.Type.ToString().ToUpper()}: {c.Name}: {c.Value}"); if (!c.Equals(MultipartContents.Last())) sb.Append(Environment.NewLine); } return sb.ToString(); } /// /// Sets multipart contents from an array of lines. /// /// The lines containing the colon-separated type, name and value of the multipart contents public void SetMultipartContents(string[] lines) { MultipartContents.Clear(); foreach(var line in lines) { try { var split = line.Split(new[] { ':' }, 3); MultipartContents.Add(new MultipartContent() { Type = (MultipartContentType)Enum.Parse(typeof(MultipartContentType), split[0].Trim(), true), Name = split[1].Trim(), Value = split[2].Trim() }); } catch { } } } #endregion /// /// Generates a random string to be used for boundary. /// /// The random 16-character string internal static string GenerateMultipartBoundary() { StringBuilder builder = new StringBuilder(); Random random = new Random(); char ch; for (int i = 0; i < 16; i++) { ch = Convert.ToChar(Convert.ToInt32(Math.Floor(26 * random.NextDouble() + 65))); builder.Append(ch); } return $"------WebKitFormBoundary{builder.ToString().ToLower()}"; } private static bool CanContainBody(HttpMethod method) { return method == HttpMethod.POST || method == HttpMethod.PUT || method == HttpMethod.DELETE; } } /// /// Represents a Multipart Content /// public class MultipartContent { /// The type of multipart content. public MultipartContentType Type { get; set; } = MultipartContentType.String; /// The name of the multipart content. public string Name { get; set; } = ""; /// The value of the multipart content (a string value or a file path). public string Value { get; set; } = ""; /// The Content-Type of the file content. public string ContentType { get; set; } = ""; } }