summaryrefslogtreecommitdiff
path: root/scraper/google.php
diff options
context:
space:
mode:
Diffstat (limited to 'scraper/google.php')
-rw-r--r--scraper/google.php505
1 files changed, 502 insertions, 3 deletions
diff --git a/scraper/google.php b/scraper/google.php
index 7ed3577..d0e90ca 100644
--- a/scraper/google.php
+++ b/scraper/google.php
@@ -824,8 +824,6 @@ class google{
$html = fread($handle, filesize("scraper/google.html"));
fclose($handle);
- $this->fuckhtml->load($html);
-
$out = [
"status" => "ok",
"spelling" => [
@@ -841,6 +839,507 @@ class google{
"news" => [],
"related" => []
];
+
+ $this->parsejavascript($html);
+
+ $containers =
+ $this->fuckhtml
+ ->getElementsByClassName(
+ $this->findstyles(
+ [
+ "background-color" => "#fff",
+ "margin-bottom" => "10px",
+ "-webkit-box-shadow" => "0 1px 6px rgba(32,33,36,0.28)",
+ "border-radius" => "8px"
+ ],
+ self::is_class
+ ),
+ "div"
+ );
+
+ foreach($containers as $container){
+
+ $this->fuckhtml->load($container);
+
+ $title =
+ $this->fuckhtml
+ ->getElementsByClassName(
+ $this->findstyles(
+ [
+ "color" => "#1967d2",
+ "font-size" => "20px",
+ "line-height" => "26px"
+ ],
+ self::is_class
+ ),
+ "div"
+ );
+
+ if(count($title) !== 0){
+
+ /*
+ Container is a web link
+ */
+ $web = [
+ "title" =>
+ $this->titledots(
+ $this->fuckhtml
+ ->getTextContent(
+ $title[0]
+ )
+ ),
+ "description" => null,
+ "url" =>
+ $this->decodeurl(
+ $this->fuckhtml
+ ->getElementsByTagName("a")
+ [0]
+ ["attributes"]
+ ["href"]
+ ),
+ "date" => null,
+ "type" => "web",
+ "thumb" => [
+ "url" => null,
+ "ratio" => null
+ ],
+ "sublink" => [],
+ "table" => []
+ ];
+
+ $container = $container["innerHTML"];
+
+ $description_container =
+ $this->fuckhtml
+ ->getElementsByClassName(
+ $this->findstyles(
+ [
+ "padding" => "12px 16px 12px"
+ ],
+ self::is_class
+ ),
+ "div"
+ )[1];
+
+ $description =
+ $description_container["innerHTML"];
+
+ // get sublinks
+ $this->fuckhtml->load($description);
+
+ $links =
+ $this->fuckhtml
+ ->getElementsByTagName("a");
+
+ $skip = true;
+ foreach($links as $link){
+
+ $description =
+ str_replace(
+ $link["outerHTML"],
+ "",
+ $description
+ );
+
+ if($skip){
+
+ $skip = false;
+ continue;
+ }
+
+ $sublink = [
+ "title" => null,
+ "description" => null,
+ "url" => null,
+ "date" => null
+ ];
+
+ $sublink["title"] =
+ $this->fuckhtml
+ ->getTextContent(
+ $link
+ );
+
+ $sublink["url"] =
+ $this->decodeurl(
+ $link
+ ["attributes"]
+ ["href"]
+ );
+
+ $web["sublink"][] = $sublink;
+ }
+
+ // get thumbnail before we call loadhtml again
+ $img =
+ $this->fuckhtml
+ ->getElementsByTagName("img");
+
+ if(count($img) !== 0){
+
+ if(
+ isset($img[0]["attributes"]["alt"]) &&
+ stripos($img[0]["attributes"]["alt"], "Video for") !== false
+ ){
+
+ // is a video thumbnail
+ $web["thumb"]["ratio"] = "16:9";
+ }else{
+
+ // is a google thumbnail
+ $web["thumb"]["ratio"] = "1:1";
+ }
+
+ $web["thumb"]["url"] =
+ $this->getimage(
+ $img[0]["attributes"]["id"]
+ );
+ }
+
+ // get table elements
+ $this->fuckhtml->load($description);
+
+ $levels =
+ $this->fuckhtml
+ ->getElementsByClassName(
+ $this->findstyles(
+ [
+ "padding-bottom" => "8px"
+ ],
+ self::is_class
+ ),
+ "div"
+ );
+
+ $additional_info = [];
+ foreach($levels as $level){
+
+ $this->fuckhtml->load($level);
+
+ $spans =
+ $this->fuckhtml
+ ->getElementsByTagName(
+ "span"
+ );
+
+ $is_rating = -2;
+
+ foreach($spans as $span){
+
+ // clean up description
+ $description =
+ str_replace(
+ $span["outerHTML"],
+ "",
+ $description
+ );
+
+ $innertext =
+ $this->fuckhtml
+ ->getTextContent(
+ $span
+ );
+
+ if($innertext == ""){ continue; }
+
+ if(
+ strtolower($innertext)
+ == "rating"
+ ){
+
+ $is_rating = -1;
+ continue;
+ }
+
+ /*
+ Parse rating object
+ */
+
+ if($is_rating >= -1){
+
+ if($span["level"] !== 1){ continue; }
+
+ $is_rating++;
+
+ // 10/10 (123)
+ if($is_rating === 0){
+
+ $innertext = explode(" ", $innertext, 2);
+
+ $web["table"]["Rating"] = $innertext[0];
+ $web["table"]["Hits"] =
+ trim(
+ str_replace(
+ [
+ "(",
+ ")"
+ ],
+ "",
+ $innertext[1]
+ )
+ );
+ continue;
+ }
+
+ // US$4.99
+ // MYR 50.00
+ // $38.34
+ // JP¥6,480
+ if($is_rating === 2){
+
+ $web["table"]["Price"] = $innertext;
+ continue;
+ }
+
+ // Android / In stock
+ if($is_rating === 4){
+
+ $web["table"]["Support"] = $innertext;
+ continue;
+ }
+
+ // ignore the rest
+ continue;
+ }
+
+ /*
+ Parse standalone text
+ */
+ $additional_info[] = $innertext;
+ }
+ }
+
+ for($i=0; $i<count($additional_info); $i++){
+
+ // @TODO
+ // generate better node names
+ $web["table"]["Info node #$i"] = $additional_info[$i];
+ }
+
+ $this->fuckhtml->load($description);
+
+ // get date node
+ $span =
+ $this->fuckhtml
+ ->getElementsByTagName(
+ "span"
+ );
+
+ if(count($span) !== 0){
+
+ $description =
+ str_replace(
+ $span[0]["outerHTML"],
+ "",
+ $description
+ );
+
+ $span =
+ strtotime(
+ $this->fuckhtml
+ ->getTextContent(
+ $span[0]
+ )
+ );
+
+ if($span){
+
+ $web["date"] = $span;
+ }
+ }
+
+ $web["description"] =
+ trim(
+ $this->fuckhtml
+ ->getTextContent(
+ $description
+ ),
+ " ·."
+ );
+
+ $out["web"][] = $web;
+
+ continue;
+ }
+
+ // check for container title header
+ $container_title =
+ $this->fuckhtml
+ ->getElementsByClassName(
+ $this->findstyles(
+ [
+ "font-weight" => "bold",
+ "font-size" => "16px",
+ "color" => "#000",
+ "margin" => "0",
+ "padding" => "12px 16px 0 16px"
+ ],
+ self::is_class
+ ),
+ "div"
+ );
+
+ if(count($container_title) !== 0){
+
+ $container_title =
+ strtolower(
+ $this->fuckhtml
+ ->getTextContent(
+ $container_title[0]
+ )
+ );
+
+ if(
+ $container_title == "related searches" ||
+ $container_title == "people also search for"
+ ){
+
+ /*
+ Parse related searches
+ */
+ $as =
+ $this->fuckhtml
+ ->getElementsByTagName("a");
+
+ foreach($as as $a){
+
+ $out["related"][] =
+ $this->fuckhtml
+ ->getTextContent($a);
+ }
+ }
+
+ continue;
+ }
+
+ /*
+ Parse image carousel
+ */
+ $title_container =
+ $this->fuckhtml
+ ->getElementsByClassName(
+ $this->findstyles(
+ [
+ "padding" => "12px 16px 12px"
+ ],
+ self::is_class
+ ),
+ "div"
+ );
+
+ if(count($title_container) !== 0){
+
+ $title_container =
+ strtolower(
+ $this->fuckhtml
+ ->getTextContent(
+ $title_container[0]
+ )
+ );
+
+ if($title_container == "imagesview all"){
+
+ /*
+ Image carousel
+ */
+ $pcitem =
+ $this->fuckhtml
+ ->getElementsByClassName(
+ "pcitem",
+ "div"
+ );
+
+ foreach($pcitem as $item){
+
+ $this->fuckhtml->load($item);
+
+ $link =
+ $this->fuckhtml
+ ->getElementsByTagName(
+ "a"
+ )[0];
+
+ parse_str(
+ parse_url(
+ $this->fuckhtml
+ ->getTextContent(
+ $link
+ ["attributes"]
+ ["href"]
+ ),
+ PHP_URL_QUERY
+ ),
+ $link
+ );
+
+ if(isset($link["tbm"])){
+
+ continue;
+ }
+
+ $image =
+ $this->fuckhtml
+ ->getElementsByTagName("img")[0];
+
+ $title =
+ $this->fuckhtml
+ ->getTextContent(
+ $image
+ ["attributes"]
+ ["alt"]
+ );
+
+ $image =
+ $this->getimage(
+ $image
+ ["attributes"]
+ ["id"]
+ );
+
+ $out["image"][] = [
+ "title" => $title,
+ "source" => [
+ [
+ "url" => $link["imgurl"],
+ "width" => (int)$link["w"],
+ "height" => (int)$link["h"]
+ ],
+ [
+ "url" => $image,
+ "width" => (int)$link["tbnw"],
+ "height" => (int)$link["tbnh"]
+ ]
+ ],
+ "url" => $link["imgrefurl"]
+ ];
+ }
+ }
+ }
+
+ /*
+ Get next page
+ */
+ $as =
+ $this->fuckhtml
+ ->getElementsByTagName("a");
+
+ foreach($as as $a){
+
+ if(
+ isset($a["attributes"]["aria-label"]) &&
+ strtolower($a["attributes"]["aria-label"]) == "next page"
+ ){
+
+ $out["npt"] =
+ $this->fuckhtml
+ ->getTextContent(
+ $a["attributes"]["href"]
+ );
+ }
+ }
+ }
+
+ return $out;
}
@@ -1163,7 +1662,7 @@ class google{
return $time;
}
- private function loadjavascriptcrap($html){
+ private function parsejavascript($html){
$this->fuckhtml->load($html);
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-16 16:49:02 +0000