diff options
Diffstat (limited to 'scraper/googlealt.php')
| -rw-r--r-- | scraper/googlealt.php | 182 |
1 files changed, 101 insertions, 81 deletions
diff --git a/scraper/googlealt.php b/scraper/googlealt.php index aa523db..d7878cf 100644 --- a/scraper/googlealt.php +++ b/scraper/googlealt.php @@ -535,6 +535,8 @@ class googlealt{ curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers); curl_setopt($curlproc, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); + + // use http2 curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); @@ -704,6 +706,43 @@ class googlealt{ // reset $this->fuckhtml->load($result_div); + }else{ + + // get the "Did you mean?" prompt + $taw = + $this->fuckhtml + ->getElementById( + "taw" + ); + + if($taw){ + + $this->fuckhtml->load($taw); + + $as = + $this->fuckhtml + ->getElementsByTagName( + "a" + ); + + if(count($as) !== 0){ + + $text = + $this->fuckhtml + ->getTextContent( + $as[0] + ); + + // @TODO implement did_you_mean + $out["spelling"] = [ + "type" => "including", + "using" => $search, + "correction" => $text + ]; + } + } + + $this->fuckhtml->load($result_div); } // @@ -895,36 +934,10 @@ class googlealt{ // get "Related Searches" and "People also search for" // $relateds = - array_merge( - $this->fuckhtml - ->getElementsByClassName( - $this->getstyle( - [ - "align-items" => "center", - "background-color" => "#28292a", - "border-radius" => "100px", - "box-sizing" => "border-box", - "display" => "flex", - "max-height" => "none", - "min-height" => "48px", - "padding-left" => "17px", - "padding-right" => "17px", - "position" => "relative" - ] - ) . " " . - $this->getstyle( - [ - "margin-left" => "8px", - "margin-right" => "8px" - ] - ), - "a" - ), - $this->fuckhtml - ->getElementsByClassName( - "wyccme", - "div" - ) + $this->fuckhtml + ->getElementsByClassName( + "wyccme", + "div" ); foreach($relateds as $related){ @@ -1354,7 +1367,7 @@ class googlealt{ "font-size" => "12px", "line-height" => "1.34", "display" => "inline-block", - "font-family" => "Google Sans,arial,sans-serif", + "font-family" => "google sans,arial,sans-serif", "padding-right" => "0", "white-space" => "nowrap" ] @@ -1401,7 +1414,7 @@ class googlealt{ "line-height" => "22px", "overflow" => "hidden", "word-break" => "break-word", - "color" => "#bdc1c6" + "color" => "#4d5156" ] ), "div" @@ -1415,12 +1428,9 @@ class googlealt{ ->getElementsByClassName( $this->getstyle( [ - "border-radius" => "10px", - "font-family" => "arial,sans-serif-medium,sans-serif", - "font-size" => "12px", - "line-height" => "16px", - "padding-block" => "2px", - "padding-inline" => "8px" + "background-color" => "rgba(0,0,0,0.6)", + "color" => "#fff", + "fill" => "#fff" ] ), "div" @@ -1433,14 +1443,6 @@ class googlealt{ ->getTextContent( $duration[0] ); - - // remove duration from description - $description[0]["innerHTML"] = - str_replace( - $duration[0]["outerHTML"], - "", - $description[0]["innerHTML"] - ); } $web["description"] = @@ -1979,7 +1981,7 @@ class googlealt{ "font-size" => "12px", "line-height" => "1.34", "display" => "inline-block", - "font-family" => "Google Sans,arial,sans-serif", + "font-family" => "google sans,arial,sans-serif", "padding-right" => "0", "white-space" => "nowrap" ] @@ -2211,7 +2213,7 @@ class googlealt{ ->getElementsByClassName( $this->getstyle( [ - "font-family" => "Google Sans,arial,sans-serif", + "font-family" => "google sans,arial,sans-serif", "font-size" => "28px", "line-height" => "36px" ] @@ -2801,7 +2803,22 @@ class googlealt{ } } - // get thumbnail + // get heading element + $heading = + $this->fuckhtml + ->getElementsByAttributeValue( + "role", + "heading", + "div" + ); + + if(count($heading) === 0){ + + // no heading, fuck this. + continue; + } + + // get thumbnail before loading heading object $image = $this->fuckhtml ->getElementsByAttributeName( @@ -2823,35 +2840,6 @@ class googlealt{ ]; } - // get title - $title = - $this->fuckhtml - ->getElementsByClassName( - $this->getstyle( - [ - "font-family" => "arial,sans-serif", - "font-size" => "16px", - "font-weight" => "400", - "line-height" => "24px" - ] - ), - "div" - ); - - if(count($title) === 0){ - - // ?? no title - continue; - } - - $title = - $this->titledots( - $this->fuckhtml - ->getTextContent( - $title[0] - ) - ); - // get duration $duration_div = $this->fuckhtml @@ -2908,6 +2896,38 @@ class googlealt{ } } + // load heading + $this->fuckhtml->load($heading[0]); + + // get title + $title = + $this->fuckhtml + ->getElementsByClassName( + $this->getstyle( + [ + "font-family" => "arial,sans-serif", + "font-size" => "16px", + "font-weight" => "400", + "line-height" => "24px" + ] + ), + "div" + ); + + if(count($title) === 0){ + + // ?? no title + continue; + } + + $title = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $title[0] + ) + ); + // get date $date_div = $this->fuckhtml @@ -3940,7 +3960,7 @@ class googlealt{ for($k=0; $k<count($values_regex[1]); $k++){ $values[trim($values_regex[1][$k])] = - trim($values_regex[2][$k]); + strtolower(trim($values_regex[2][$k])); } $names = explode(",", $matches[1][$i]); @@ -3971,7 +3991,7 @@ class googlealt{ foreach($this->styles[":root"] as $key => $value){ - $this->css_colors[$value] = $key; + $this->css_colors[$value] = strtolower($key); } } } @@ -4206,7 +4226,7 @@ class googlealt{ throw new Exception("Failed to get HTML"); } - //$html = file_get_contents("scraper/google-video.html"); + //$html = file_get_contents("scraper/google.html"); $response = $this->parsepage($html, "videos", $search, $proxy, $params); $out = [ |