fixed brave spelling and other things

1 files changed, 547 insertions, 1314 deletions

diff --git a/scraper/brave.php b/scraper/brave.php
index 0a73158..3a1c0ce 100644
--- a/scraper/brave.php
+++ b/scraper/brave.php
@@ -74,6 +74,13 @@ class brave{
 					"older" => [
 						"display" => "Older than",
 						"option" => "_DATE"
+					],
+					"spellcheck" => [
+						"display" => "Spellcheck",
+						"option" => [
+							"no" => "No",
+							"yes" => "Yes"
+						]
 					]
 				];
 				break;
@@ -203,7 +210,7 @@ class brave{
 			$q = json_decode($this->nextpage->get($get["npt"], "web"), true);
 			
 			$search = $q["q"];
-			$q["spellcheck"] = 0;
+			$q["spellcheck"] = "0";
 			
 			$nsfw = $q["nsfw"];
 			unset($q["nsfw"]);
@@ -215,6 +222,7 @@ class brave{
 			
 			// get _GET data instead
 			$search = $get["s"];
+			
 			if(strlen($search) === 0){
 				
 				throw new Exception("Search term is empty!");
@@ -229,6 +237,7 @@ class brave{
 			$country = $get["country"];
 			$older = $get["older"];
 			$newer = $get["newer"];
+			$spellcheck = $get["spellcheck"];
 			
 			$q = [
 				"q" => $search
@@ -264,12 +273,18 @@ class brave{
 				
 				$q["tf"] = "{$newer}to{$older}";
 			}
+			
+			// spellcheck
+			if($spellcheck == "no"){
+				
+				$q["spellcheck"] = "0";
+			}
 		}
-		/*
+		
 		$handle = fopen("scraper/brave.html", "r");
 		$html = fread($handle, filesize("scraper/brave.html"));
 		fclose($handle);
-		*/
+		/*
 		try{
 			$html =
 				$this->get(
@@ -283,7 +298,7 @@ class brave{
 			
 			throw new Exception("Could not fetch search page");
 		}
-		
+		*/
 		$out = [
 			"status" => "ok",
 			"spelling" => [
@@ -308,1458 +323,510 @@ class brave{
 		*/
 		$nextpage =
 			$this->fuckhtml
-			->getElementsByClassName(
-				"btn ml-15",
-				"a"
-			);
-		
-		if(count($nextpage) !== 0){
-			
-			preg_match(
-				'/offset=([0-9]+)/',
-				$this->fuckhtml->getTextContent($nextpage[0]["attributes"]["href"]),
-				$nextpage
-			);
-				
-			$q["offset"] = (int)$nextpage[1];
-			$q["nsfw"] = $nsfw;
-			$q["country"] = $country;
-			
-			$out["npt"] =	
-				$this->nextpage->store(
-					json_encode($q),
-					"web"
-				);
-		}
-		
-		/*
-			Get discussions (and append them to web results)
-		*/
-		
-		// they're loaded using javascript!!
-		$discussion =
-			$this->fuckhtml
 			->getElementById(
-				"js-discussions",
-				"script"
+				"pagination",
+				"div"
 			);
 		
-		if(
-			$discussion &&
-			isset($discussion["attributes"]["data"])
-		){
+		if($nextpage){
 			
-			$discussion =
-				json_decode(
-					$this->fuckhtml
-					->getTextContent(
-						$discussion["attributes"]["data"]
-					),
-					true
-				);
+			$this->fuckhtml->load($nextpage);
+			
+			$nextpage =
+				$this->fuckhtml
+				->getElementsByClassName("btn", "a");
 			
-			foreach($discussion["results"] as $result){
+			if(count($nextpage) !== 0){
 				
-				$data = [
-					"title" => $this->titledots($result["title"]),
-					"description" => null,
-					"url" => $result["url"],
-					"date" => null,
-					"type" => "web",
-					"thumb" => [
-						"url" => null,
-						"ratio" => null
-					],
-					"sublink" => [],
-					"table" => []
-				];
+				$nextpage =
+					$nextpage[count($nextpage) - 1];
 				
-				// description
-				$data["description"] =
-					$this->limitstrlen(
-						$this->limitwhitespace(
-							$this->titledots(
-								$this->fuckhtml->getTextContent(
-									$result["description"]
-								)
-							)
+				if(
+					strtolower(
+						$this->fuckhtml
+						->getTextContent(
+							$nextpage
 						)
-					);
-				
-				if($result["age"] != ""){
-					$data["date"] = strtotime($result["age"]);
-				}
-				
-				// populate table
-				
-				if($result["data"]["num_answers"] != ""){
-					$data["table"]["Replies"] = (int)$result["data"]["num_answers"];
-				}
-				
-				if($result["data"]["score"] != ""){
-					
-					$score = explode("|", $result["data"]["score"]);
+					) == "next"
+				){
 					
-					if(count($score) === 2){
-						
-						$score = ((int)$score[1]) . " (" . trim($score[0]) . ")";
-					}else{
+					preg_match(
+						'/offset=([0-9]+)/',
+						$this->fuckhtml->getTextContent($nextpage["attributes"]["href"]),
+						$nextpage
+					);
 						
-						$score = (int)$score[0];
-					}
-					
-					$data["table"]["Votes"] = $score;
-				}
-				
-				if($result["thumbnail"] != ""){
+					$q["offset"] = (int)$nextpage[1];
+					$q["nsfw"] = $nsfw;
+					$q["country"] = $country;
 					
-					$data["thumb"]["url"] = $result["thumbnail"];
-					$data["thumb"]["ratio"] = "16:9";
+					$out["npt"] =
+						$this->nextpage->store(
+							json_encode($q),
+							"web"
+						);
 				}
-				
-				$out["web"][] = $data;
 			}
 		}
 		
-		/*
-			Get related searches
-		*/
-		$faq =
+		$this->fuckhtml->load($html);
+		
+		$script_disc =
 			$this->fuckhtml
-			->getElementById("js-faq", "script");
+			->getElementsByTagName(
+				"script"
+			);
 		
-		if(
-			$faq &&
-			isset($faq["attributes"]["data"])
-		){
+		$grep = [];
+		foreach($script_disc as $discs){
 			
-			$faq =
-				json_decode(
-					$this->fuckhtml
-					->getTextContent(
-						$faq["attributes"]["data"]
-					),
-					true
-				);
+			preg_match(
+				'/const data ?= ?(\[{.*}]);/',
+				$discs["innerHTML"],
+				$grep
+			);
 			
-			foreach($faq["items"] as $related){
+			if(isset($grep[1])){
 				
-				$out["related"][] = $related["question"];
+				break;
 			}
 		}
 		
-		/*
-			Get spelling autocorrect
-		*/
-		$altered =
-			$this->fuckhtml
-			->getElementById("altered-query", "div");
-		
-		if($altered){
-			
-			$this->fuckhtml->load($altered);
-			
-			$altered =
-				$this->fuckhtml
-				->getElementsByTagName("a");
+		if(!isset($grep[1])){
 			
-			if(count($altered) === 2){
-				
-				$out["spelling"] = [
-					"type" => "including",
-					"using" =>
-						$this->fuckhtml
-						->getTextContent($altered[0]),
-					"correction" =>
-						$this->fuckhtml
-						->getTextContent($altered[1])
-				];
-			}
-			
-			$this->fuckhtml->load($html);
+			throw new Exception("Could not get data JS");
 		}
+			
+		$data =
+			$this->fuckhtml
+			->parseJsObject(
+				$grep[1]
+			);
+		unset($grep);
+		
+		$data = $data[1]["data"]["body"]["response"];
 		
 		/*
 			Get web results
 		*/
-		$resulthtml =
-			$this->fuckhtml
-			->getElementById(
-				"results",
-				"div"
-			);
-		
-		$this->fuckhtml->load($resulthtml);
-		$items = 0;
-		foreach(
-			$this->fuckhtml
-			->getElementsByClassName("snippet fdb")
-			as $result
-		){
+		if(!isset($data["web"]["results"])){
 			
-			$data = [
-				"title" => null,
-				"description" => null,
-				"url" => null,
-				"date" => null,
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
+			return $out;
+		}
+		
+		//$has_so_answer = false;
+		
+		foreach($data["web"]["results"] as $result){
 			
-			if(
-				isset($result["attributes"]["data-type"]) &&
-				$result["attributes"]["data-type"] == "ad"
-			){
+			if(isset($result["thumbnail"])){
+				
+				$thumb = [
+					"ratio" => $result["thumbnail"]["logo"] == "false" ? "16:9" : "1:1",
+					"url" => $result["thumbnail"]["original"]
+				];
+			}else{
 				
-				// is an ad, skip
-				continue;
+				$thumb = [
+					"ratio" => null,
+					"url" => null
+				];
 			}
 			
-			$this->fuckhtml->load($result);
-			
-			/*
-				Get title
-			*/
-			$title =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"snippet-title",
-					"span"
-				);
-			
-			if(count($title) === 0){
+			$sublink = [];
+			if(isset($result["cluster"])){
 				
-				// encountered AI summarizer
-				// or misspelling indicator @TODO
-				continue;
+				foreach($result["cluster"] as $cluster){
+					
+					$sublink[] = [
+						"title" => $this->titledots($cluster["title"]),
+						"description" =>
+							$this->titledots(
+								$this->fuckhtml
+								->getTextContent(
+									$cluster["description"]
+								)
+							),
+						"url" => $cluster["url"],
+						"date" => null
+					];
+				}
 			}
 			
-			if(isset($title[0]["attributes"]["title"])){
+			// parse table elements
+			$table = [];
+			
+			// product
+			$ref = null;
+			
+			if(isset($result["product"])){
 				
-				$data["title"] =
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$title[0]["attributes"]["title"]
-						)
-					);
-			}else{
+				$ref = &$result["product"];
+			}elseif(isset($result["creative_work"])){
 				
-				$data["title"] =
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$title[0]
-						)
-					);
+				$ref = &$result["creative_work"];
 			}
 			
-			/*
-				Get description
-			*/
-			$description =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"snippet-description",
-					"p"
-				);
-			
-			if(count($description) !== 0){
-				$data["description"] =
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$description[0]
-						)
-					);
-				
-				// also check for thumbnail in here
-				$img =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"thumb",
-						"img"
-					);
+			if($ref !== null){
 				
-				if(count($img) !== 0){
+				if(isset($ref["offers"])){
 					
-					$data["thumb"] = [
-						"url" => $this->unshiturl($img[0]["attributes"]["src"]),
-						"ratio" => "16:9"
-					];
-				}else{
-					
-					// might be a video thumbnail wrapper?
-					$wrapper =
-						$this->fuckhtml
-						->getElementsByClassName(
-							"video-thumb",
-							"a"
-						);
-					
-					if(count($wrapper) !== 0){
-						
-						// we found a video
-						$this->fuckhtml->load($wrapper[0]);
+					foreach($ref["offers"] as $offer){
 						
-						$img =
-							$this->fuckhtml
-							->getElementsByTagName("img");
+						$price = null;
 						
-						$data["thumb"] = [
-							"url" => $this->unshiturl($img[0]["attributes"]["src"]),
-							"ratio" => "16:9"
-						];
+						if(isset($offer["price"])){
+							
+							if((float)$offer["price"] == 0){
+								
+								$price = "Free";
+							}else{
+								
+								$price = $offer["price"];
+							}
+						}
 						
-						// get the video length, if its there
-						$duration =
-							$this->fuckhtml
-							->getElementsByClassName(
-								"duration",
-								"div"
-							);
+						if($price !== "Free"){
+							if(isset($offer["priceCurrency"])){
+								
+								$price .= " " . $offer["priceCurrency"];
+							}
+						}
 						
-						if(count($duration) !== 0){
+						if($price !== null){
 							
-							$data["table"]["Duration"] = $duration[0]["innerHTML"];
+							$table["Price"] = trim($price);
 						}
-						
-						// reset html load
-						$this->fuckhtml->load($result);
 					}
 				}
 				
-			}else{
-				
-				// is a steam/shop listing
-				$description_alt =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"text-sm",
-						"div"
-					);
-				
-				if(count($description_alt) !== 0){
+				if(isset($ref["rating"])){
 					
-					switch($description_alt[0]["attributes"]["class"]){
+					$rating = null;
+					if(isset($ref["rating"]["ratingValue"])){
+						
+						$rating = $ref["rating"]["ratingValue"];
 						
-						case "text-sm text-gray":
-						case "description text-sm":
+						if(isset($ref["rating"]["bestRating"])){
 							
-							$data["description"] =
-								$this->titledots(
-									$this->fuckhtml
-									->getTextContent(
-										$description_alt[0]
-									)
-								);
-							break;
+							$rating .= "/" . $ref["rating"]["bestRating"];
+						}
 					}
 					
-					// get table sublink
-					$sublink =
-						$this->fuckhtml
-						->getElementsByClassName(
-							"r-attr text-sm",
-							"div"
-						);
-					
-					if(count($sublink) !== 0){
+					if(isset($ref["rating"]["reviewCount"])){
 						
-						$this->tablesublink($sublink, $data);
-					}
-					
-					// check for thumb element
-					$data["thumb"] = $this->getimagelinkfromstyle("thumb");
-				}else{
-					
-					// ok... finally...
-					// maybe its the instant answer thingy
-					$answer =
-						$this->fuckhtml
-						->getElementsByClassName("answer");
-					
-					if(count($answer) !== 0){
+						$isnull = $rating === null ? false : true;
 						
-						$data["description"] =
-							$this->titledots(
-								$this->fuckhtml
-								->getTextContent($answer[0])
-							);
-					}
-				}
-			}
-			
-			// finally, fix brave's date format sucking balls
-			$data["description"] = explode(" - ", $data["description"], 2);
-			
-			if(count($data["description"]) === 0){
-				
-				// nothing to do
-				$data["description"] = $data["description"][0];
-			}else{
-				
-				// attempt to parse
-				$time = strtotime($data["description"][0]);
-				
-				if($time !== false){
-					
-					// got response
-					$data["date"] = $time;
-					
-					array_shift($data["description"]);
-				}
-								
-				// merge back
-				$data["description"] =
-					implode(" - ", $data["description"]);
-			}
-			
-			/*
-				Check content type
-			*/
-			$content_type =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"content-type",
-					"span"
-				);
-			
-			if(count($content_type) !== 0){
-				
-				$data["type"] =
-					strtolower($this->fuckhtml->getTextContent($content_type[0]));
-			}
-			
-			/*
-				Check subtext table thingy
-			*/
-			$table_items =
-				array_merge(
-					$this->fuckhtml
-					->getElementsByClassName(
-						"item-attributes",
-						"div"
-					),
-					$this->fuckhtml
-					->getElementsByClassName(
-						"r",
-						"div"
-					)
-				);
-			
-			/*
-				DIV: item-attributes
-			*/
-			if(count($table_items) !== 0){
-				
-				foreach($table_items as $table){
-					
-					$this->fuckhtml->load($table);
-					
-					$span =
-						$this->fuckhtml
-						->getElementsByClassName(
-							"text-sm",
-							"*"
-						);
-					
-					foreach($span as $item){
+						if($isnull){
+							
+							$rating .= " (";
+						}
 						
-						$item =
-							explode(
-								":",
-								$this->fuckhtml->getTextContent(preg_replace('/\n/', " ", $item["innerHTML"])),
-								2
-							);
+						$rating .= number_format($ref["rating"]["reviewCount"]) . " hits";
 						
-						if(count($item) === 2){
+						if($isnull){
 							
-							$data["table"][trim($item[0])] = trim($this->limitwhitespace($item[1]));
+							$rating .= ")";
 						}
 					}
+					
+					if($rating !== null){
+						
+						$table["Rating"] = $rating;
+					}
 				}
-				
-				$this->fuckhtml->load($result);
 			}
 			
-			// get video sublinks
-			$table_items =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"snippet-description published-time",
-					"p"
-				);
-			
-			if(count($table_items) !== 0){
+			// video
+			if(isset($result["video"])){
 				
-				$table_items =
-					explode(
-						'<span class="mr-15"></span>',
-						$table_items[0]["innerHTML"],
-						2
-					);
-				if(count($table_items) === 2){
-					
-					$item2 = [];
-					
-					$item2[] = explode(":", $this->fuckhtml->getTextContent($table_items[0]));
+				foreach($result["video"] as $key => $value){
 					
-					if(trim($table_items[1]) != ""){
-						$item2[] = explode(":", $this->fuckhtml->getTextContent($table_items[1]));
-					}
-					
-					foreach($item2 as $it){
+					if(is_string($result["video"][$key]) === false){
 						
-						$data["table"][trim($it[0])] = trim($it[1]);
+						continue;
 					}
+					
+					$table[ucfirst($key)] = $value;
 				}
 			}
 			
 			/*
-				Get URL
+				Get StackOverflow answers
 			*/
-			$data["url"] =
-				$this->fuckhtml->getTextContent(
-					$this->fuckhtml
-					->getElementsByTagName("a")
-					[0]
-					["attributes"]
-					["href"]
-				);
-			
-			/*
-				Get sublinks
-			*/
-			$sublinks_elems =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"snippet",
-					"div"
-				);
-			
-			$sublinks = [];
-			
-			foreach($sublinks_elems as $sublink){
-				
-				$this->fuckhtml->load($sublink);
-				
-				$a =
-					$this->fuckhtml
-					->getElementsByTagName("a")[0];
-				
-				$title =
-					$this->fuckhtml
-					->getTextContent($a);
-				
-				$url = $a["attributes"]["href"];
-				
-				$description =
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$this->fuckhtml
-							->getElementsByTagName("p")[0]
-						)
-					);
-				
-				$sublinks[] = [
-					"title" => $title,
-					"date" => null,
-					"description" => $description,
-					"url" => $url
-				];
-			}
-			
+			// commented out since it also returns alot of garbage
 			/*
-				Get smaller sublinks
-			*/
-			$sublinks_elems =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"deep-link",
-					"a"
-				);
-			
-			foreach($sublinks_elems as $sublink){
-				
-				$sublinks[] = [
-					"title" => $this->fuckhtml->getTextContent($sublink),
-					"date" => null,
-					"description" => null,
-					"url" => $sublink["attributes"]["href"]
-				];
-			}
-			
-			// append sublinks to $data !!
-			$data["sublink"] = $sublinks;
-			
-			// append first result to start of $out["web"]
-			// other results are after
-			if($items === 0){
-				
-				$out["web"] = [$data, ...$out["web"]];
-			}else{
-				
-				$out["web"][] = $data;
-			}
-			$items++;
-		}
-		
-		/*
-			Get news
-		*/
-		$this->fuckhtml->load($resulthtml);
-		$news_carousel = $this->fuckhtml->getElementById("news-carousel");
-		
-		$this->fuckhtml->load($news_carousel);
-		
-		if($news_carousel){
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"card fdb",
-					"a"
-				);
-			
-			foreach($a as $news){
-				
-				$this->fuckhtml->load($news);
-				
-				$out["news"][] = [
-					"title" =>
-						$this->titledots(
-							$this->fuckhtml
-							->getTextContent(
-								$this->fuckhtml
-								->getElementsByClassName(
-									"title",
-									"div"
-								)[0]
-							)
-						),
-					"description" => null,
-					"date" =>
-						strtotime(
-							$this->fuckhtml
-							->getTextContent(
-								$this->fuckhtml
-								->getElementsByClassName(
-									"card-footer__timestamp",
-									"span"
-								)[0]
-							)
-						),
-					"thumb" => $this->getimagelinkfromstyle("img-bg"),
-					"url" => $this->fuckhtml->getTextContent($news["attributes"]["href"])
-				];
-			}
-		}
-		
-		
-		
-		/*
-			Get videos
-		*/
-		$this->fuckhtml->load($resulthtml);
-		$news_carousel = $this->fuckhtml->getElementById("video-carousel");
-		
-		$this->fuckhtml->load($news_carousel);
-		
-		if($news_carousel){
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"card fdb",
-					"a"
-				);
-			
-			foreach($a as $video){
-				
-				$this->fuckhtml->load($video);
+			if(
+				$has_so_answer === false &&
+				isset($result["qa"])
+			){
 				
-				$date = null;
+				$has_so_answer = true;
+				$answer = $this->stackoverflow_parse($result["qa"]["answer"]["text"]);
 				
-				$date_o =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"text-gray text-xs",
-						"span"
-					);
-				
-				if(count($date_o) !== 0){
+				if(isset($result["qa"]["answer"]["author"])){
 					
-					$date =
-						strtotime(
-							$this->fuckhtml
-							->getTextContent(
-								$date_o[0]
-							)
-						);
+					$answer[] = [
+						"type" => "quote",
+						"value" => "Answer from " . $result["qa"]["answer"]["author"]
+					];
 				}
 				
-				$out["video"][] = [
+				$out["answer"][] = [
 					"title" =>
-						$this->titledots(
-							$this->fuckhtml
-							->getTextContent(
-								$this->fuckhtml
-								->getElementsByClassName(
-									"title",
-									"div"
-								)[0]
-							)
+						$this->fuckhtml
+						->getTextContent(
+							$result["qa"]["question"]
 						),
-					"description" => null,
-					"date" => $date,
-					"duration" => null,
-					"views" => null,
-					"thumb" => $this->getimagelinkfromstyle("img-bg"),
-					"url" => $this->fuckhtml->getTextContent($video["attributes"]["href"])
+					"description" => $answer,
+					"url" => $result["url"],
+					"thumb" => null,
+					"table" => [],
+					"sublink" => []
 				];
-			}
+			}*/
+			
+			$out["web"][] = [
+				"title" =>
+					$this->titledots(
+						$result["title"]
+					),
+				"description" =>
+					$this->titledots(
+						$this->fuckhtml
+						->getTextContent(
+							$result["description"]
+						)
+					),
+				"url" => $result["url"],
+				"date" => isset($result["age"]) ? strtotime($result["age"]) : null,
+				"type" => "web",
+				"thumb" => $thumb,
+				"sublink" => $sublink,
+				"table" => $table
+			];
 		}
 		
-		
 		/*
-			Get DEFINITION snippet
+			Get spelling autocorrect
 		*/
-		$this->fuckhtml->load($html);
-		$infobox = $this->fuckhtml->getElementById("rh-definitions", "div");
-		
-		if($infobox !== false){
+		if(
+			isset($data["query"]["bo_altered_diff"][0][0]) &&
+			$data["query"]["bo_altered_diff"][0][0] == "true"
+		){
 			
-			$answer = [
-				"title" => null,
-				"description" => [],
-				"url" => null,
-				"thumb" => null,
-				"table" => [],
-				"sublink" => []
+			$out["spelling"] = [
+				"type" => "including",
+				"using" => $data["query"]["bo_altered_diff"][0][1],
+				"correction" => $get["s"]
 			];
-			
-			$this->fuckhtml->load($infobox);
-			
-			$answer["title"] =
-				$this->fuckhtml
-				->getTextContent(
-					$this->fuckhtml
-					->getElementsByClassName(
-						"header",
-						"h5"
-					)[0]
-				);
-			
-			$sections =
-				$this->fuckhtml
-				->getElementsByTagName("section");
-			
-			$i = -1;
-			foreach($sections as $section){
-				
-				$this->fuckhtml->load($section);
-				$items =
-					$this->fuckhtml
-					->getElementsByTagName("*");
-				
-				$li = 1;
-				$pronounce = false;
-				foreach($items as $item){
-					
-					switch($item["tagName"]){
-						
-						case "h6":
-							
-							if(
-								isset($item["attributes"]["class"]) &&
-								$item["attributes"]["class"] == "h6 pronunciation"
-							){
-								
-								if($pronounce){
-									
-									break;
-								}
-								
-								$answer["description"][] = [
-									"type" => "quote",
-									"value" =>
-										$this->fuckhtml
-										->getTextContent(
-											$item
-										)
-								];
-								
-								$answer["description"][] =
-									[
-										"type" => "audio",
-										"url" => "https://search.brave.com/api/rhfetch?rhtype=definitions&word={$answer["title"]}&source=ahd-5"
-									];
-								
-								$pronounce = true;
-								$i = $i + 2;
-								break;
-							}
-							
-							$answer["description"][] = [
-								"type" => "title",
-								"value" =>
-									$this->fuckhtml
-									->getTextContent(
-										$item
-									)
-							];
-							$i++;
-							break;
-						
-						case "li":
-							
-							if(
-								$i !== -1 &&
-								$answer["description"][$i]["type"] == "text"
-							){
-								
-								$answer["description"][$i]["value"] .=
-									"\n" . $li . ". " .
-									$this->fuckhtml
-									->getTextContent(
-										$item
-									);
-								
-							}else{
-								$answer["description"][] = [
-									"type" => "text",
-									"value" =>
-										$li . ". " .
-										$this->fuckhtml
-										->getTextContent(
-											$item
-										)
-								];
-								$i++;
-							}
-							$li++;
-							break;
-						
-						case "a":
-							$answer["url"] =
-								$this->fuckhtml
-								->getTextContent(
-									$item["attributes"]["href"]
-								);
-							break;
-					}
-				}
-			}
-			
-			$out["answer"][] = $answer;
 		}
 		
-		
 		/*
-			Get instant answer
+			Get wikipedia heads
 		*/
-		$this->fuckhtml->load($html);
-		$infobox = $this->fuckhtml->getElementById("infobox", "div");
-		
-		if($infobox !== false){
-			
-			$answer = [
-				"title" => null,
-				"description" => [],
-				"url" => null,
-				"thumb" => null,
-				"table" => [],
-				"sublink" => []
-			];
+		if(isset($data["infobox"]["results"][0])){
 			
-			$this->fuckhtml->load($infobox);
-			$div = $this->fuckhtml->getElementsByTagName("div");
-			
-			/*
-				Get small description
-			*/
-			$small_desc =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"infobox-description",
-					$div
-				);
-			
-			if(count($small_desc) !== 0){
-				
-				$answer["description"][] = [
-					"type" => "quote",
-					"value" =>
-						$this->fuckhtml
-						->getTextContent(
-							$small_desc[0]
-						)
-				];
-			}
-			
-			/*
-				Get title + url
-			*/
-			$title =
-				$this->fuckhtml
-				->getElementsByClassName("infobox-title", "a");
-			
-			if(count($title) !== 0){
+			foreach($data["infobox"]["results"] as $info){
 				
-				$answer["title"] =
-					$this->fuckhtml
-					->getTextContent(
-						$title[0]
-					);
-				
-				$answer["url"] =
-					$this->fuckhtml
-					->getTextContent(
-						$title[0]["attributes"]["href"]
-					);
-			}
-			
-			/*
-				Get thumbnail
-			*/
-			$thumb = $this->getimagelinkfromstyle("thumb");
-			
-			if($thumb["url"] !== null){
-				
-				$answer["thumb"] = $thumb["url"];
-			}
-			
-			/*
-				Get table
-			*/
-			$title =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"infobox-attr-header",
-					"div"
-				);
-			
-			$rowhtml = $infobox;
-			
-			if(count($title) >= 2){
-				
-				$rowhtml =
-					explode(
-						$title[1]["outerHTML"],
-						$infobox["innerHTML"],
-						2
-					)[0];
-			}
-			
-			$this->fuckhtml->load($rowhtml);
-			
-			$rows =
-				$this->fuckhtml
-				->getElementsByClassName("infobox-attr", "div");
-			
-			foreach($rows as $row){
-				
-				if(!isset($row["innerHTML"])){
+				if($info["subtype"] == "code"){
 					
-					continue;
-				}
-				
-				$this->fuckhtml->load($row);
-				$span =
-					$this->fuckhtml
-					->getElementsByTagName("span");
-				
-				if(count($span) === 2){
+					$description =
+						$this->stackoverflow_parse($info["data"]["answer"]["text"]);
 					
-					$answer["table"][
-						$this->fuckhtml->getTextContent($span[0])
-					] = str_replace("\n", ", ", $this->fuckhtml->getTextContent($span[1], true));
-				}
-			}
-			
-			$this->fuckhtml->load($infobox);
-			
-			/*
-				Parse stackoverflow answers
-			*/
-			$code =
-				$this->fuckhtml
-				->getElementById("codebox-answer", $div);
-			
-			if($code){
-				
-				// this might be standalone text with no paragraphs, check for that
-				$author =
-					$this->fuckhtml
-					->getElementById("author");
-				
-				$desc_tmp =
-					str_replace(
-						$author["outerHTML"],
-						"",
-						$code["innerHTML"]
-					);
-				
-				$this->fuckhtml->load($desc_tmp);
-				$code =
-					$this->fuckhtml
-					->getElementsByTagName("*");
-				
-				if(count($code) === 0){
-					
-					$answer["description"][] = [
-						"type" => "text",
-						"value" =>
-							$this->fuckhtml
-							->getTextContent(
-								$desc_tmp
-							)
-					];
-					
-					$answer["description"][] = [
-						"type" => "quote",
-						"value" =>
-							$this->fuckhtml
-							->getTextContent(
-								$author
-							)
-					];
-				}else{
-					
-					$i = 0;
-					
-					foreach($code as $snippet){
+					if(isset($info["data"]["answer"]["author"])){
 						
-						switch($snippet["tagName"]){
-							
-							case "p":
-								$this->fuckhtml->load($snippet["innerHTML"]);
-								
-								$codetags =
-									$this->fuckhtml
-									->getElementsByTagName("*");
-								
-								$tmphtml = $snippet["innerHTML"];
-								
-								foreach($codetags as $tag){
-									
-									if(!isset($tag["outerHTML"])){
-										
-										continue;
-									}
-									
-									$tmphtml =
-										explode(
-											$tag["outerHTML"],
-											$tmphtml,
-											2
-										);
-									
-									$value = $this->fuckhtml->getTextContent($tmphtml[0], false, false);
-									$this->appendtext($value, $answer["description"], $i);
-									
-									$type = null;
-									switch($tag["tagName"]){
-										
-										case "code": $type = "inline_code"; break;
-										case "em": $type = "italic"; break;
-										case "blockquote": $type = "quote"; break;
-										default: $type = "text";
-									}
-									
-									if($type !== null){
-										$value = $this->fuckhtml->getTextContent($tag, false, true);
-										
-										if(trim($value) != ""){
-											
-											if(
-												$i !== 0 &&
-												$type == "title"
-											){
-												
-												$answer["description"][$i - 1]["value"] = rtrim($answer["description"][$i - 1]["value"]);
-											}
-											
-											$answer["description"][] = [
-												"type" => $type,
-												"value" => $value
-											];
-											$i++;
-										}
-									}
-									
-									if(count($tmphtml) === 2){
-										
-										$tmphtml = $tmphtml[1];
-									}else{
-										
-										break;
-									}
-								}
-								
-								if(is_array($tmphtml)){
-									
-									$tmphtml = $tmphtml[0];
-								}
-								
-								if(strlen($tmphtml) !== 0){
-									
-									$value = $this->fuckhtml->getTextContent($tmphtml, false, false);
-									$this->appendtext($value, $answer["description"], $i);
-								}
-								break;
-							
-							case "pre":
-								
-								switch($answer["description"][$i - 1]["type"]){
-									
-									case "text":
-									case "italic":
-										$answer["description"][$i - 1]["value"] = rtrim($answer["description"][$i - 1]["value"]);
-										break;
-								}
-								
-								$answer["description"][] =
-									[
-										"type" => "code",
-										"value" =>
-											rtrim(
-												$this->fuckhtml
-												->getTextContent(
-													$snippet,
-													true,
-													false
-												)
-											)
-									];
-								$i++;
-								
-								break;
-							
-							case "ol":
-								$o = 0;
-								
-								$this->fuckhtml->load($snippet);
-								$li =
-									$this->fuckhtml
-									->getElementsByTagName("li");
-								
-								foreach($li as $elem){
-									$o++;
-									
-									$this->appendtext(
-										$o . ". " .
-										$this->fuckhtml
-										->getTextContent(
-											$elem
-										),
-										$answer["description"],
-										$i
-									);
-								}
-								break;
-						}
+						$description[] = [
+							"type" => "quote",
+							"value" => "Answer from " . $info["data"]["answer"]["author"]
+						];
 					}
+				}else{
+					
+					$description = [];
 					
 					if(
-						$i !== 0 &&
-						$answer["description"][$i - 1]["type"] == "text"
+						isset($info["description"]) &&
+						$info["description"] != ""
 					){
-						
-						$answer["description"][$i - 1]["value"] = rtrim($answer["description"][$i - 1]["value"]);
-					}
-					
-					if($author){
-						
-						$answer["description"][] = [
+						$description[] = [
 							"type" => "quote",
-							"value" => $this->fuckhtml->getTextContent($author)
+							"value" => $info["description"]
 						];
 					}
-				}
-			}else{
-				
-				/*
-					Get normal description
-				*/
-				$description =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"mb-6",
-						"div"
-					);
-				
-				if(count($description) !== 0){
 					
-					$answer["description"][] =
-						[
+					if(
+						isset($info["long_desc"]) &&
+						$info["long_desc"] != ""
+					){
+						$description[] = [
 							"type" => "text",
-							"value" =>
-								$this->titledots(
-									preg_replace(
-										'/ Wikipedia$/',
-										"",
-										$this->fuckhtml
-										->getTextContent(
-											$description[0]
-										)
-									)
-								)
+							"value" => $this->titledots($info["long_desc"])
 						];
+					}
+				}
+				
+				$table = [];
+				if(isset($info["attributes"])){
 					
-					$ratings =
-						$this->fuckhtml
-						->getElementById("ratings");
-					
-					if($ratings){
-						
-						$this->fuckhtml->load($ratings);
-						
-						$ratings =
-							$this->fuckhtml
-							->getElementsByClassName(
-								"flex-hcenter mb-10",
-								"div"
-							);
+					foreach($info["attributes"] as $row){
 						
-						$answer["description"][] = [
-							"type" => "title",
-							"value" => "Ratings"
-						];
-						
-						foreach($ratings as $rating){
-							
-							$this->fuckhtml->load($rating);
-							
-							$num =
-								$this->fuckhtml
-								->getTextContent(
-									$this->fuckhtml
-									->getElementsByClassName(
-										"r-num",
-										"div"
-									)[0]
-								);
-							
-							$href =
-								$this->fuckhtml
-								->getElementsByClassName(
-									"mr-10",
-									"a"
-								)[0];
-							
-							$votes =
-								$this->fuckhtml
-								->getTextContent(
-									$this->fuckhtml
-									->getElementsByClassName(
-										"text-sm",
-										"span"
-									)[0]
-								);
-							
-							$c = count($answer["description"]) - 1;
+						if(
+							$row[1] == "null" &&
+							count($table) !== 0
+						){
 							
-							if(
-								$c !== -1 &&
-								$answer["description"][$c]["type"] == "text"
-							){
-								
-								$answer["description"][$c]["value"] .= $num . " ";
-							}else{
-								
-								$answer["description"][] = [
-									"type" => "text",
-									"value" => $num . " "
-								];
-							}
-							
-							$answer["description"][] = [
-								"type" => "link",
-								"value" => $this->fuckhtml->getTextContent($href),
-								"url" => $this->fuckhtml->getTextContent($href["attributes"]["href"])
-							];
+							break;
+						}
+						
+						if($row[1] == "null"){
 							
-							$answer["description"][] = [
-								"type" => "text",
-								"value" => " (" . $votes . ")\n"
-							];
+							continue;
 						}
+						
+						$table[
+							$this->fuckhtml->getTextContent($row[0])
+						] =
+							$this->fuckhtml->getTextContent($row[1]);
 					}
 				}
-			}
-			
-			/*
-				Get sublinks
-			*/
-			$this->fuckhtml->load($infobox);
-			
-			$profiles =
-				$this->fuckhtml
-				->getElementById("profiles");
-			
-			if($profiles){
-				$profiles =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"chip",
-						"a"
-					);
 				
-				foreach($profiles as $profile){
-					
-					$name = $this->fuckhtml->getTextContent($profile["attributes"]["title"]);
+				$sublink = [];
+				if(isset($info["profiles"])){
 					
-					if(strtolower($name) == "steampowered"){
+					foreach($info["profiles"] as $row){
+						
+						$name = $this->fuckhtml->getTextContent($row["name"]);
+						
+						if(strtolower($name) == "steampowered"){
+							
+							$name = "Steam";
+						}
 						
-						$name = "Steam";
+						$sublink[
+							$this->fuckhtml->getTextContent($name)
+						] =
+							$this->fuckhtml->getTextContent($row["url"]);
 					}
-					
-					$answer["sublink"][$name] =
-						$this->fuckhtml->getTextContent($profile["attributes"]["href"]);
 				}
-			}
-			
-			$actors =
-				$this->fuckhtml
-				->getElementById("panel-movie-cast");
-			
-			if($actors){
 				
-				$this->fuckhtml->load($actors);
+				$out["answer"][] = [
+					"title" => $this->fuckhtml->getTextContent($info["title"]),
+					"description" => $description,
+					"url" => $info["url"],
+					"thumb" => isset($info["images"][0]["original"]) ? $info["images"][0]["original"] : null,
+					"table" => $table,
+					"sublink" => $sublink
+				];
 				
-				$actors =
-					$this->fuckhtml
-					->getElementsByClassName("card");
+				break; // only iterate once, we get garbage most of the time
+			}
+		}
+		
+		/*
+			Get videos
+		*/
+		if(isset($data["videos"]["results"])){
+			
+			foreach($data["videos"]["results"] as $video){
 				
-				$answer["description"][] = [
-					"type" => "title",
-					"value" => "Cast"
+				$out["video"][] = [
+					"title" => $this->titledots($video["title"]),
+					"description" => $this->titledots($video["description"]),
+					"date" => isset($video["age"]) ? strtotime($video["age"]) : null,
+					"duration" => isset($video["video"]["duration"]) ? $this->hms2int($video["video"]["duration"]) : null,
+					"views" => null,
+					"thumb" =>
+						isset($video["thumbnail"]["src"]) ?
+						[
+							"ratio" => "16:9",
+							"url" => $this->unshiturl($video["thumbnail"]["src"])
+						] :
+						[
+							"ratio" => null,
+							"url" => null
+						],
+					"url" => $video["url"]
 				];
-				
-				foreach($actors as $actor){
-					
-					$this->fuckhtml->load($actor);
-					
-					$answer["description"][] = [
-						"type" => "text",
-						"value" =>
-							$this->fuckhtml
-							->getTextContent(
-								$this->fuckhtml
-								->getElementsByClassName("card-body")
-								[0]
-							)
-					];
-					
-					$answer["description"][] = [
-						"type" => "image",
-						"url" => $this->getimagelinkfromstyle("person-thumb")["url"]
-					];
-				}
 			}
+		}
+		
+		/*
+			Get news
+		*/
+		if(isset($data["news"]["results"])){
 			
-			$out["answer"][] = $answer;
+			foreach($data["news"]["results"] as $news){
+				
+				$out["news"][] = [
+					"title" => $this->titledots($news["title"]),
+					"description" => $this->titledots($news["description"]),
+					"date" => isset($news["age"]) ? strtotime($news["age"]) : null,
+					"thumb" =>
+						isset($video["thumbnail"]["src"]) ?
+						[
+							"ratio" => "16:9",
+							"url" => $this->unshiturl($video["thumbnail"]["src"])
+						] :
+						[
+							"ratio" => null,
+							"url" => null
+						],
+					"url" => $news["url"]
+				];
+			}
 		}
 		
 		/*
-			Get actor standalone thingy
+			Get discussions
 		*/
-		$this->fuckhtml->load($resulthtml);
-		$actors =
-			$this->fuckhtml
-			->getElementById("predicate-entity");
+		$disc_out = [];
 		
-		if($actors){
-			
-			$this->fuckhtml->load($actors);
+		if(isset($data["discussions"]["results"])){
 			
-			$cards =
-				$this->fuckhtml
-				->getElementsByClassName("card");
-			
-			$url =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"disclaimer",
-					"div"
-				)[0];
-			
-			$this->fuckhtml->load($url);
-			
-			$url =
-				$this->fuckhtml
-				->getTextContent(
-					$this->fuckhtml
-					->getElementsByTagName("a")
-					[0]
-					["attributes"]
-					["href"]
-				);
-			
-			$this->fuckhtml->load($actors);
-			
-			$answer = [
-				"title" =>
-					$this->fuckhtml
-					->getTextContent(
-						$this->fuckhtml
-						->getElementsByClassName(
-							"entity",
-							"span"
-						)[0]
-					) . " (Cast)",
-				"description" => [],
-				"url" => $url,
-				"sublink" => [],
-				"thumb" => null,
-				"table" => []
-			];
-			
-			foreach($cards as $card){
+			foreach($data["discussions"]["results"] as $disc){
 				
-				$this->fuckhtml->load($card);
+				$table = [];
 				
-				$answer["description"][] = [
-					"type" => "title",
-					"value" =>
-						$this->fuckhtml
-						->getTextContent(
-							$this->fuckhtml
-							->getElementsByClassName(
-								"title"
-							)[0]
-						)
-				];
+				if(isset($disc["data"]["num_votes"])){
+					
+					$table["Votes"] = (int)$disc["data"]["num_votes"];
+				}
 				
-				$answer["description"][] = [
-					"type" => "text",
-					"value" =>
-						$this->fuckhtml
-						->getTextContent(
-							$this->fuckhtml
-							->getElementsByClassName(
-								"text-xs desc"
-							)[0]
-						)
-				];
+				if(isset($disc["data"]["num_answers"])){
+					
+					$table["Comments"] = (int)$disc["data"]["num_answers"];
+				}
 				
-				$answer["description"][] = [
-					"type" => "image",
-					"url" => $this->getimagelinkfromstyle("img-bg")["url"]
+				$disc_out[] = [
+					"title" =>
+						$this->titledots(
+							$disc["title"]
+						),
+					"description" =>
+						$this->limitstrlen(
+							$this->titledots(
+								$this->fuckhtml
+								->getTextContent(
+									$disc["description"]
+								)
+							)
+						),
+					"url" => $disc["url"],
+					"date" => isset($disc["age"]) ? strtotime($disc["age"]) : null,
+					"type" => "web",
+					"thumb" => [
+						"ratio" => null,
+						"url" => null
+					],
+					"sublink" => [],
+					"table" => $table
 				];
 			}
-			
-			$out["answer"][] = $answer;
 		}
 		
+		// append discussions at position 2
+		array_splice($out["web"], 1, 0, $disc_out);
+		
 		return $out;
 	}
 	
@@ -2124,6 +1191,172 @@ class brave{
 		return $out;
 	}
 	
+	private function stackoverflow_parse($html){
+		
+		$i = 0;
+		$answer = [];
+		
+		$this->fuckhtml->load($html);
+		
+		foreach(
+			$this->fuckhtml->getElementsByTagName("*")
+			as $snippet
+		){
+			
+			switch($snippet["tagName"]){
+				
+				case "p":
+					$this->fuckhtml->load($snippet["innerHTML"]);
+					
+					$codetags =
+						$this->fuckhtml
+						->getElementsByTagName("*");
+					
+					$tmphtml = $snippet["innerHTML"];
+					
+					foreach($codetags as $tag){
+						
+						if(!isset($tag["outerHTML"])){
+							
+							continue;
+						}
+						
+						$tmphtml =
+							explode(
+								$tag["outerHTML"],
+								$tmphtml,
+								2
+							);
+						
+						$value = $this->fuckhtml->getTextContent($tmphtml[0], false, false);
+						$this->appendtext($value, $answer, $i);
+						
+						$type = null;
+						switch($tag["tagName"]){
+							
+							case "code": $type = "inline_code"; break;
+							case "em": $type = "italic"; break;
+							case "blockquote": $type = "quote"; break;
+							default: $type = "text";
+						}
+						
+						if($type !== null){
+							$value = $this->fuckhtml->getTextContent($tag, false, true);
+							
+							if(trim($value) != ""){
+								
+								if(
+									$i !== 0 &&
+									$type == "title"
+								){
+									
+									$answer[$i - 1]["value"] = rtrim($answer[$i - 1]["value"]);
+								}
+								
+								$answer[] = [
+									"type" => $type,
+									"value" => $value
+								];
+								$i++;
+							}
+						}
+						
+						if(count($tmphtml) === 2){
+							
+							$tmphtml = $tmphtml[1];
+						}else{
+							
+							break;
+						}
+					}
+					
+					if(is_array($tmphtml)){
+						
+						$tmphtml = $tmphtml[0];
+					}
+					
+					if(strlen($tmphtml) !== 0){
+						
+						$value = $this->fuckhtml->getTextContent($tmphtml, false, false);
+						$this->appendtext($value, $answer, $i);
+					}
+					break;
+				
+				case "img":
+					$answer[] = [
+						"type" => "image",
+						"url" =>
+							$this->fuckhtml
+							->getTextContent(
+								$tag["attributes"]["src"]
+							)
+					];
+					$i++;
+					break;
+				
+				case "pre":
+					
+					switch($answer[$i - 1]["type"]){
+						
+						case "text":
+						case "italic":
+							$answer[$i - 1]["value"] = rtrim($answer[$i - 1]["value"]);
+							break;
+					}
+					
+					$answer[] =
+						[
+							"type" => "code",
+							"value" =>
+								rtrim(
+									$this->fuckhtml
+									->getTextContent(
+										$snippet,
+										true,
+										false
+									)
+								)
+						];
+					$i++;
+					
+					break;
+				
+				case "ol":
+					$o = 0;
+					
+					$this->fuckhtml->load($snippet);
+					$li =
+						$this->fuckhtml
+						->getElementsByTagName("li");
+					
+					foreach($li as $elem){
+						$o++;
+						
+						$this->appendtext(
+							$o . ". " .
+							$this->fuckhtml
+							->getTextContent(
+								$elem
+							),
+							$answer,
+							$i
+						);
+					}
+					break;
+			}
+		}
+		
+		if(
+			$i !== 0 &&
+			$answer[$i - 1]["type"] == "text"
+		){
+			
+			$answer[$i - 1]["value"] = rtrim($answer[$i - 1]["value"]);
+		}
+		
+		return $answer;
+	}
+	
 	private function hms2int($time){
 		
 		$parts = explode(":", $time, 3);