diff options
| -rw-r--r-- | README.md | 72 | ||||
| -rw-r--r-- | about.php | 130 | ||||
| -rw-r--r-- | api.txt | 289 | ||||
| -rw-r--r-- | api/index.php | 10 | ||||
| -rw-r--r-- | api/v1/images.php | 25 | ||||
| -rw-r--r-- | api/v1/index.php | 10 | ||||
| -rw-r--r-- | api/v1/news.php | 25 | ||||
| -rw-r--r-- | api/v1/videos.php | 25 | ||||
| -rw-r--r-- | api/v1/web.php | 30 | ||||
| -rw-r--r-- | audio.php | 19 | ||||
| -rw-r--r-- | banner/aves.png | bin | 0 -> 86557 bytes | |||
| -rw-r--r-- | banner/aves_2.png | bin | 0 -> 51960 bytes | |||
| -rw-r--r-- | banner/bibblebop.png | bin | 0 -> 42488 bytes | |||
| -rw-r--r-- | banner/birds birds birdsw_4.jpg | bin | 0 -> 9643 bytes | |||
| -rw-r--r-- | banner/birds_birds_birdsw.jpg | bin | 0 -> 6641 bytes | |||
| -rw-r--r-- | banner/birds_birds_birdsw_2.jpg | bin | 0 -> 6959 bytes | |||
| -rw-r--r-- | banner/birds_birds_birdsw_3.jpg | bin | 0 -> 6231 bytes | |||
| -rw-r--r-- | banner/deek.png | bin | 0 -> 1971 bytes | |||
| -rw-r--r-- | banner/deekchat.gif | bin | 0 -> 3699 bytes | |||
| -rw-r--r-- | banner/eagle.png | bin | 0 -> 17033 bytes | |||
| -rw-r--r-- | banner/eagle2.png | bin | 0 -> 5236 bytes | |||
| -rw-r--r-- | banner/eagle3.jpg | bin | 0 -> 11607 bytes | |||
| -rw-r--r-- | banner/eddd_1.png | bin | 0 -> 71856 bytes | |||
| -rw-r--r-- | banner/eddd_2.png | bin | 0 -> 33842 bytes | |||
| -rw-r--r-- | banner/eddd_3.png | bin | 0 -> 51809 bytes | |||
| -rw-r--r-- | banner/gnuwu.png | bin | 0 -> 22305 bytes | |||
| -rw-r--r-- | banner/gnuwu_2.png | bin | 0 -> 47351 bytes | |||
| -rw-r--r-- | banner/horse.png | bin | 0 -> 69595 bytes | |||
| -rw-r--r-- | banner/linucks.jpg | bin | 0 -> 63271 bytes | |||
| -rw-r--r-- | banner/real_nig_3.jpg | bin | 0 -> 67812 bytes | |||
| -rw-r--r-- | banner/sec.png | bin | 0 -> 60673 bytes | |||
| -rw-r--r-- | banner/tagmachine.png | bin | 0 -> 35288 bytes | |||
| -rw-r--r-- | favicon.ico | bin | 0 -> 393 bytes | |||
| -rw-r--r-- | favicon.php | 362 | ||||
| -rw-r--r-- | icons/lolcat.ca.png | bin | 0 -> 1408 bytes | |||
| -rw-r--r-- | images.php | 99 | ||||
| -rw-r--r-- | index.php | 14 | ||||
| -rw-r--r-- | lib/bingcache-todo-fix.php | 144 | ||||
| -rw-r--r-- | lib/classic.png | bin | 0 -> 7623 bytes | |||
| -rw-r--r-- | lib/curlproxy.php | 652 | ||||
| -rw-r--r-- | lib/favicon404.png | bin | 0 -> 807 bytes | |||
| -rw-r--r-- | lib/frontend.php | 1282 | ||||
| -rw-r--r-- | lib/fuckhtml.php | 361 | ||||
| -rw-r--r-- | lib/img404.png | bin | 0 -> 4549 bytes | |||
| -rw-r--r-- | lib/nextpage.php | 106 | ||||
| -rw-r--r-- | lib/type-todo.php | 132 | ||||
| -rw-r--r-- | news.php | 96 | ||||
| -rw-r--r-- | opensearch.xml | 9 | ||||
| -rw-r--r-- | proxy.php | 130 | ||||
| -rw-r--r-- | robots.txt | 28 | ||||
| -rw-r--r-- | scraper/brave.php | 2287 | ||||
| -rw-r--r-- | scraper/ddg.php | 2722 | ||||
| -rw-r--r-- | scraper/google.php | 1562 | ||||
| -rw-r--r-- | scraper/marginalia.php | 242 | ||||
| -rw-r--r-- | scraper/mojeek.php | 1182 | ||||
| -rw-r--r-- | scraper/wiby.php | 244 | ||||
| -rw-r--r-- | scraper/yandex.php | 530 | ||||
| -rw-r--r-- | scraper/youtube.php | 1723 | ||||
| -rw-r--r-- | settings.php | 316 | ||||
| -rw-r--r-- | sitemap.xml | 19 | ||||
| -rw-r--r-- | static/client.js | 682 | ||||
| -rw-r--r-- | static/icon/amazon.png | bin | 0 -> 2485 bytes | |||
| -rw-r--r-- | static/icon/appstore.png | bin | 0 -> 2841 bytes | |||
| -rw-r--r-- | static/icon/facebook.png | bin | 0 -> 1575 bytes | |||
| -rw-r--r-- | static/icon/gamespot.png | bin | 0 -> 2434 bytes | |||
| -rw-r--r-- | static/icon/github.png | bin | 0 -> 1250 bytes | |||
| -rw-r--r-- | static/icon/googleplay.png | bin | 0 -> 1469 bytes | |||
| -rw-r--r-- | static/icon/imdb.png | bin | 0 -> 2626 bytes | |||
| -rw-r--r-- | static/icon/instagram.png | bin | 0 -> 3309 bytes | |||
| -rw-r--r-- | static/icon/itunes.png | bin | 0 -> 3016 bytes | |||
| -rw-r--r-- | static/icon/microsoft.png | bin | 0 -> 2147 bytes | |||
| -rw-r--r-- | static/icon/quora.png | bin | 0 -> 1716 bytes | |||
| -rw-r--r-- | static/icon/reddit.png | bin | 0 -> 2009 bytes | |||
| -rw-r--r-- | static/icon/rottentomatoes.png | bin | 0 -> 1084 bytes | |||
| -rw-r--r-- | static/icon/sciencedirect.png | bin | 0 -> 585 bytes | |||
| -rw-r--r-- | static/icon/soundcloud.png | bin | 0 -> 1289 bytes | |||
| -rw-r--r-- | static/icon/spotify.png | bin | 0 -> 2114 bytes | |||
| -rw-r--r-- | static/icon/steam.png | bin | 0 -> 1875 bytes | |||
| -rw-r--r-- | static/icon/twitter.png | bin | 0 -> 1361 bytes | |||
| -rw-r--r-- | static/icon/w3html.png | bin | 0 -> 1542 bytes | |||
| -rw-r--r-- | static/icon/website.png | bin | 0 -> 2854 bytes | |||
| -rw-r--r-- | static/icon/wikipedia.png | bin | 0 -> 1363 bytes | |||
| -rw-r--r-- | static/icon/youtube.png | bin | 0 -> 2363 bytes | |||
| -rw-r--r-- | static/style.css | 1176 | ||||
| -rw-r--r-- | template/header.html | 28 | ||||
| -rw-r--r-- | template/home.html | 36 | ||||
| -rw-r--r-- | template/images.html | 7 | ||||
| -rw-r--r-- | template/search.html | 16 | ||||
| -rw-r--r-- | videos.php | 241 | ||||
| -rw-r--r-- | web.php | 496 |
90 files changed, 17559 insertions, 0 deletions
diff --git a/README.md b/README.md new file mode 100644 index 0000000..039d6a0 --- /dev/null +++ b/README.md @@ -0,0 +1,72 @@ +# 4get +4get is a metasearch engine that doesn't suck (they live in our walls!) + +## About 4get +https://4get.ca/about + +## Try it out +https://4get.ca + +# Setup +Login as root. + +```sh +apt install apache2 certbot php-dom php-imagick imagemagick php-curl curl php-apcu git libapache2-mod-php python3-certbot-apache +service apache2 start +a2enmod rewrite +``` + +For all of the files in `/etc/apache2/sites-enabled/`, you must apply the following changes: +- Uncomment `ServerName` directive, put your domain name there +- Change `ServerAdmin` to your email +- Change `DocumentRoot` to `/var/www/html/4get` +- Change `ErrorLog` and `CustomLog` directives to log stuff out to `/dev/null/` + +Now open `/etc/apache2/apache2.conf` and change `ErrorLog` and `CustomLog` directives to have `/dev/null/` as a value + +This *should* disable logging completely, but I'm not 100% sure since I sort of had to troubleshoot alot of shit while writing this. So after we're done check if `/var/log/apache2/*` contains any personal info, and if it does, call me retarded trough email exchange. + +Blindly run the following shit + +```sh +cd /var/www/html +git clone https://git.lolcat.ca/lolcat/4get +cd 4get +mkdir icons +chmod 777 -R icons/ +``` + +Restart the service for good measure... `service apache2 restart` + +## Setup encryption +I'm schizoid (as you should) so I'm gonna setup 4096bit key encryption. To complete this step, you need a domain or subdomain in your possession. Make sure that the DNS shit for your domain has propagated properly before continuing, because certbot is a piece of shit that will error out the ass once you reach 5 attempts under an hour. + +```sh +certbot --apache --rsa-key-size 4096 -d www.yourdomain.com -d yourdomain.com +``` +When it asks to choose a vhost, choose the option with "HTTPS" listed. Don't setup HTTPS for tor, we don't need it (it doesn't even work anyways with let's encrypt) + +Edit `000-default-le-ssl.conf` + +Add this at the end: +```xml +<Directory /var/www/html/4get> + RewriteEngine On + RewriteCond %{REQUEST_FILENAME}.php -f + RewriteRule (.*) $1.php [L] + Options Indexes FollowSymLinks + AllowOverride All + Require all granted +</Directory> +``` + +Now since this file is located in `/etc/apache2/sites-enabled/`, you must change all of the logging shit as to make it not log anything, like we did earlier. + +Restart again +```sh +service apache2 restart +``` + +You'll probably want to setup a tor address at this point, but I'm too lazy to put instructions here. + +Ok bye!!! diff --git a/about.php b/about.php new file mode 100644 index 0000000..fdc4812 --- /dev/null +++ b/about.php @@ -0,0 +1,130 @@ +<?php + +include "lib/frontend.php"; +$frontend = new frontend(); + +echo + '<!DOCTYPE html>' . + '<html lang="en">' . + '<head>' . + '<meta http-equiv="Content-Type" content="text/html;charset=utf-8">' . + '<title>About</title>' . + '<link rel="stylesheet" href="/static/style.css">' . + '<meta name="viewport" content="width=device-width,initial-scale=1">' . + '<meta name="robots" content="index,follow">' . + '<link rel="icon" type="image/x-icon" href="/favicon.ico">' . + '<meta name="description" content="4get.ca: About">' . + '<link rel="search" type="application/opensearchdescription+xml" title="4get" href="/opensearch.xml">' . + '</head>' . + '<body class="' . $frontend->getthemeclass(false) . 'about">'; + +$left = + '<a href="/" class="link">< Go back</a> + + <h1>Set as default search engine</h1> + <a href="#firefox"><h2 id="firefox">On Firefox and other Gecko based browsers</h2></a> + To set this as your default search engine on Firefox, right click the URL bar and select <div class="code-inline">Add "4get"</div>. Then, visit <a href="about:preferences#search" target="_BLANK" class="link">about:preferences#search</a> and select <div class="code-inline">4get</div> in the dropdown menu. + + <a href="#chrome"><h2 id="chrome">On Chromium and Blink based browsers</h2></a> + Right click the URL bar and click <div class="code-inline">Manage search engines and site search</div>, or visit <a href="chrome://settings/searchEngines" target="_BLANK" class="link">chrome://settings/searchEngines</a>. Then, create a new entry under <div class="code-inline">Search engines</div> and fill in the following details: + + <table> + <tr> + <td><b>Field</b></td> + <td><b>Value</b></td> + </tr> + <tr> + <td>Search engine</td> + <td>4get</td> + </tr> + <tr> + <td>Shortcut</td> + <td>4get.ca</td> + </tr> + <tr> + <td>URL with %s in place of query</td> + <td>https://4get.ca/web?q=%s</td> + </tr> + </table> + + Once that\'s done, click <div class="code-inline">Save</div>. Then, on the right handside of the newly created entry, open the dropdown menu and select <div class="code-inline">Make default</div>. + + <a href="#other-browsers"><h2 id="other-browsers">Other browsers</h2></a> + Get a real browser. + + <h1>Frequently asked questions</h1> + <a href="#what-is-this"><h2 id="what-is-this">What is this?</h2></a> + This is a metasearch engine that gets results from other engines, and strips away all of the tracking parameters and Microsoft/globohomo bullshit they add. Most of the other alternatives to Google jack themselves off about being ""privacy respecting"" or whatever the fuck but it always turns out to be a total lie, and I just got fed up with their shit honestly. Alternatives like Searx or YaCy all fucking sucks so I made my own thing. + + <a href="#goal"><h2 id="goal">My goal</h2></a> + Provide users with a privacy oriented, extremely lightweight, ad free, free as in freedom (and free beer!) way to search for documents around the internet, with minimal, optional javascript code. My long term goal would be to build my own index (that doesn\'t suck) and provide users with an unbiased search engine, with no political inclinations. + + <a href="#logs"><h2 id="logs">Do you keep logs?</h2></a> + I store data temporarly to get the next page of results. This might include search queries, tokens and other parameters. These parameters are encrypted using <div class="code-inline">aes-256-gcm</div> on the serber, for which I give you a key (also known internally as <div class="code-inline">npt</div> token). When you make a request to get the next page, you supply the token, the data is decrypted and the request is fulfilled. This encrypted data is deleted after 7 minutes, or after it\'s used, whichever comes first.<br><br> + + I <b>don\'t</b> log IP addresses, user agents, or anything else. The <div class="code-inline">npt</div> tokens are the only thing that are stored (in RAM, mind you), temporarly, encrypted. + + <a href="#information-sharing"><h2 id="information-sharing">Do you share information with third parties?</h2></a> + Your search queries and supplied filters are shared with the scraper you chose (so I can get the search results, duh). I don\'t share anything else (that means I don\'t share your IP address, location, or anything of this kind). There is no way that site can know you\'re the one searching for something, <u>unless you send out a search query that de-anonymises you.</u> For example, a search query like "hello my full legal name is jonathan gallindo and i want pictures of cloacas" would definitively blow your cover. 4get doesn\'t contain ads or any third party javascript applets or trackers. I don\'t profile you, and quite frankly, I don\'t give a shit about what you search on there.<br><br> + + TL;DR assume those websites can see what you search for, but can\'t see who you are (unless you\'re really dumb). + + <a href="#hosting"><h2 id="hosting">Where is this website hosted?</h2></a> + This website is hosted on a Contabo shitbox in the United States. + + <a href="#keyboard-shortcuts"><h2 id="keyboard-shortcuts">Keyboard shortcuts?</h2></a> + Use <div class="code-inline">/</div> to focus the search box.<br><br> + + When the image viewer is open, you can use the following keybinds:<br> + <div class="code-inline">Up</div>, <div class="code-inline">Down</div>, <div class="code-inline">Left</div>, <div class="code-inline">Right</div> to rotate the image.<br> + <div class="code-inline">CTRL+Up</div>, <div class="code-inline">CTRL+Down</div>, <div class="code-inline">CTRL+Left</div>, <div class="code-inline">CTRL+Right</div> to mirror the image.<br> + <div class="code-inline">Escape</div> to exit the image viewer. + + <a href="#instances"><h2 id="instances">Instances</h2></a> + 4get is open source, anyone can create their own 4get instance! If you wish to add your website to this list, please <a href="https://lolcat.ca/contact">contact me</a>. + + <table> + <tr> + <td>Name</td> + <td>Address</td> + </tr> + <tr> + <td>4get</td> + <td><a href="https://4get.ca">4get.ca</a><a href="http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion/">(tor)</a></td> + </tr> + </table> + + <a href="#schizo"><h2 id="schizo">How can I trust you?</h2></a> + You just sort of have to take my word for it right now. If you\'d rather trust yourself instead of me (I believe in you!!), all of the code on this website is available trough my <a href="https://git.lolcat.ca/lolcat" class="link">git page</a> for you to host on your own machines. Just a reminder: if you\'re the sole user of your instance, it doesn\'t take immense brain power for Microshit to figure out you basically just switched IP addresses. Invite your friends to use your instance! + + <a href="#contact"><h2 id="contact">I want to report abuse or have erotic roleplay trough email</h2></a> + I don\'t know about that second part but if you want to talk to me, just drop me an email...<br><br> + + <b>Message to all DMCA enforcers:</b> I don\'t host any of the content. Everything you see here is <u>proxied</u> trough my shitbox with no moderation. Please reach out to the people hosting the infringing content instead.<br><br> + + <a href="https://lolcat.ca/contact" rel="dofollow" class="link">Click here to contact me!</a><br><br> + + <a href="https://validator.w3.org/nu/?doc=https%3A%2F%2F4get.ca" title="W3 Valid!"> + <img src="/static/icon/w3html.png" alt="Valid W3C HTML 4.01" width="88" height="31"> + </a>'; + +// trim out whitespace +$left = explode("\n", $left); + +$out = ""; + +foreach($left as $line){ + + $out .= trim($line); +} + +echo + $frontend->load( + "search.html", + [ + "class" => "", + "right-left" => "", + "right-right" => "", + "left" => $out + ] + ); @@ -0,0 +1,289 @@ + __ __ __ + / // / ____ ____ / /_ + / // /_/ __ `/ _ \/ __/ + /__ __/ /_/ / __/ /_ + /_/ \__, /\___/\__/ + /____/ + + + Welcome to the 4get API documentation + + ++ Terms of use + Do NOT misuse the API. Misuses can include... :: + + 1. Serp SEO scanning + 2. Intensive scraping + 3. Any other activity that isn't triggered by a human + 4. Illegal activities in Canada + 5. Constant "test" queries while developping your program + (please cache the API responses!) + + + Examples of good uses of the API :: + + 1. A chatroom bot that presents users with search results + 2. Personal use + 3. Any other activity that is initiated by a human + + + If you wish to engage in the activities listed under "misuses", feel + free to download the source code of the project and running 4get + under your own terms. Please respect the terms of use listed here so + that this website may be available to all in the far future. + + Get your instance running here :: + https://git.lolcat.ca/lolcat/4get + + Thanks! + + ++ Decode the data + All payloads returned by the API are encoded in the JSON format. If + you don't know how to tackle the problem, maybe programming is not + for you. + + All of the endpoints use the GET method. + + ++ Check if an API call was successful + All API responses come with an array index named "status". If the + status is something else than the string "ok", something went wrong. + + The HTTP code will always be 200 as to not cause issues with CORS. + + ++ Get the next page of results + All API responses come with an array index named "nextpage". To get + the next page of results, you must make another API call with &npt. + + Example :: + + + First API call + /api/v1/web?s=higurashi + + + Second API call + /api/v1/web?npt=ddg1._rJ2hWmYSjpI2hsXWmYajJx < ... > + + You shouldn't specify the search term, only the &npt parameter + suffices. + + The first part of the token before the dot (ddg1) refers to an + array position on the serber's memory. The second part is an + encryption key used to decode the data at that position. This way, + it is impossible to supply invalid pagination data and it is + impossible for a 4get operator to peek at the private data of the + user after a request has been made. + + The tokens will expire as soon as they are used or after a 7 minutes + inactivity period, whichever comes first. + + ++ Beware of null values! + Most fields in the API responses can return "null". You don't need + to worry about unset values. + + ++ API Parameters + To construct a valid request, you can use the 4get web interface + to craft a valid request, and replace "/web" with "/api/v1/web". + + ++ "date" and "time" parameters + "date" always refer to a calendar date. + "time" always refer to the duration of some media. + + They are both integers that uses seconds as its unit. The "date" + parameter specifies the number of seconds that passed since January + 1st 1970. + + + ______ __ _ __ + / ____/___ ____/ /___ ____ (_)___ / /______ + / __/ / __ \/ __ / __ \/ __ \/ / __ \/ __/ ___/ + / /___/ / / / /_/ / /_/ / /_/ / / / / / /_(__ ) + /_____/_/ /_/\__,_/ .___/\____/_/_/ /_/\__/____/ + /_/ + ++ /api/v1/web + + &extendedsearch + When using the ddg(DuckDuckGo) scraper, you may make use of the + &extendedsearch parameter. If you need rich answer data from + additional sources like StackOverflow, music lyrics sites, etc., + you need to specify the value of (string)"true". + + The default value is "false" for API calls. + + + + Parse the "spelling" + The array index named "spelling" contains 3 indexes :: + + spelling: + type: "including" + using: "4chan" + correction: '"4cha"' + + + The "type" may be any of these 3 values. When rendering the + autocorrect text inside your application, it should look like + what follows right after the parameter value :: + + no_correction <Empty> + including Including results for %using%. Did you mean + %correction%? + + not_many Not many results for %using%. Did you mean + %correction%? + + + As of right now, the "spelling" is only available on + "/api/v1/web". + + + + Parse the "answer" + The array index named "answer" may contain a list of multiple + answers. The array index "description" contains a linear list of + nodes that can help you construct rich formatted data inside of + your application. The structure is similar to the one below: + + answer: + 0: + title: "Higurashi" + description: + 0: + type: "text" + value: "Higurashi is a great show!" + 1: + type: "quote" + value: "Source: my ass" + + + Each "description" node contains an array index named "type". + Here is a list of them: + + text + + title + italic + + quote + + code + inline_code + link + + image + + audio + + + Each individual node prepended with a "+" should be prepended by + a newline when constructing the rendered description object. + + There are some nodes that differ from the type-value format. + Please parse them accordingly :: + + + link + type: "link" + url: "https://lolcat.ca" + value: "Visit my website!" + + + + image + type: "image" + url: "https://lolcat.ca/static/pixels.png" + + + + audio + type: "audio" + url: "https://lolcat.ca/static/whatever.mp3" + + + The array index named "table" is an associative array. You can + loop over the data using this PHP code, for example :: + + foreach($table as $website_name => $url){ // ... + + + The rest of the JSON is pretty self explanatory. + + ++ /api/v1/images + All images are contained within "image". The structure looks like + below :: + + image: + 0: + title: "My awesome Higurashi image" + source: + 0: + url: "https://lolcat.ca/static/profile_pix.png" + width: 400 + height: 400 + 1: + url: "https://lolcat.ca/static/pixels.png" + width: 640 + height: 640 + 2: + url: "https://tse1.mm.bing.net/th?id=OIP.VBM3BQg + euf0-xScO1bl1UgHaGG" + width: 194 + height: 160 + + + The last image of the "source" array is always the thumbnail, and is + a good fallback to use when other sources fail to load. There can be + more than 1 source; this is especially true when using the Yandex + scraper, but beware of captcha rate limits. + + ++ /api/v1/videos + The "time" parameter for videos may be set to "_LIVE". For live + streams, the amount of people currently watching is passed in + "views". + + ++ /api/v1/news + Just make a request to "/api/v1/news?s=elon+musk". The payload + has nothing special about it and is very self explanatory, just like + the endpoint above. + + ++ /favicon + Get the favicon for a website. The only parameter is "s", and must + include the protocol. + + Example :: + + /favicon?s=https://lolcat.ca + + + If we had to revert to using Google's favicon cache, it will throw + an error in the X-Error header field. If Google's favicon cache + also failed to return an image, or if you're too retarded to specify + a valid domain name, a default placeholder image will be returned + alongside the "404" HTTP error code. + + ++ /proxy + Get a proxied image. Useful if you don't want to leak your user's IP + address. The parameters are "i" for the image link and "s" for the + size. + + Acceptable "s" parameters: + + portrait 90x160 + landscape 160x90 + square 90x90 + thumb 236x180 + cover 207x270 + original <Original resolution> + + You can also ommit the "s" parameter if you wish to view the + original image. When an error occurs, an "X-Error" header field + is set. + + ++ /audio + Get a proxied audio file. Does not support "Range" headers, as it's + only used to proxy small files. + + The parameter is "s" for the audio link. + + ++ Appendix + If you have any questions or need clarifications, please send an + email my way to will at lolcat.ca diff --git a/api/index.php b/api/index.php new file mode 100644 index 0000000..dae86ab --- /dev/null +++ b/api/index.php @@ -0,0 +1,10 @@ +<?php + +header("Content-Type: application/json"); +http_response_code(404); + +echo json_encode( + [ + "status" => "Unknown endpoint" + ] +); diff --git a/api/v1/images.php b/api/v1/images.php new file mode 100644 index 0000000..e05ba26 --- /dev/null +++ b/api/v1/images.php @@ -0,0 +1,25 @@ +<?php + +header("Content-Type: application/json"); + +chdir("../../"); + +include "lib/frontend.php"; +$frontend = new frontend(); + +[$scraper, $filters] = $frontend->getscraperfilters( + "images", + isset($_GET["scraper"]) ? $_GET["scraper"] : null +); + +$get = $frontend->parsegetfilters($_GET, $filters); + +try{ + echo json_encode( + $scraper->image($get) + ); + +}catch(Exception $e){ + + echo json_encode(["status" => $e->getMessage()]); +} diff --git a/api/v1/index.php b/api/v1/index.php new file mode 100644 index 0000000..dae86ab --- /dev/null +++ b/api/v1/index.php @@ -0,0 +1,10 @@ +<?php + +header("Content-Type: application/json"); +http_response_code(404); + +echo json_encode( + [ + "status" => "Unknown endpoint" + ] +); diff --git a/api/v1/news.php b/api/v1/news.php new file mode 100644 index 0000000..7e24247 --- /dev/null +++ b/api/v1/news.php @@ -0,0 +1,25 @@ +<?php + +header("Content-Type: application/json"); + +chdir("../../"); + +include "lib/frontend.php"; +$frontend = new frontend(); + +[$scraper, $filters] = $frontend->getscraperfilters( + "news", + isset($_GET["scraper"]) ? $_GET["scraper"] : null +); + +$get = $frontend->parsegetfilters($_GET, $filters); + +try{ + echo json_encode( + $scraper->news($get) + ); + +}catch(Exception $e){ + + echo json_encode(["status" => $e->getMessage()]); +} diff --git a/api/v1/videos.php b/api/v1/videos.php new file mode 100644 index 0000000..60c105a --- /dev/null +++ b/api/v1/videos.php @@ -0,0 +1,25 @@ +<?php + +header("Content-Type: application/json"); + +chdir("../../"); + +include "lib/frontend.php"; +$frontend = new frontend(); + +[$scraper, $filters] = $frontend->getscraperfilters( + "videos", + isset($_GET["scraper"]) ? $_GET["scraper"] : null +); + +$get = $frontend->parsegetfilters($_GET, $filters); + +try{ + echo json_encode( + $scraper->video($get) + ); + +}catch(Exception $e){ + + echo json_encode(["status" => $e->getMessage()]); +} diff --git a/api/v1/web.php b/api/v1/web.php new file mode 100644 index 0000000..7895183 --- /dev/null +++ b/api/v1/web.php @@ -0,0 +1,30 @@ +<?php + +header("Content-Type: application/json"); + +chdir("../../"); + +include "lib/frontend.php"; +$frontend = new frontend(); + +[$scraper, $filters] = $frontend->getscraperfilters( + "web", + isset($_GET["scraper"]) ? $_GET["scraper"] : null +); + +$get = $frontend->parsegetfilters($_GET, $filters); + +if(!isset($_GET["extendedsearch"])){ + + $get["extendedsearch"] = "no"; +} + +try{ + echo json_encode( + $scraper->web($get) + ); + +}catch(Exception $e){ + + echo json_encode(["status" => $e->getMessage()]); +} diff --git a/audio.php b/audio.php new file mode 100644 index 0000000..bb018da --- /dev/null +++ b/audio.php @@ -0,0 +1,19 @@ +<?php + +if(!isset($_GET["s"])){ + + http_response_code(404); + header("X-Error: No SOUND(s) provided!"); + die(); +} + +include "lib/curlproxy.php"; +$proxy = new proxy(); + +try{ + + $proxy->stream_linear_audio($_GET["s"]); +}catch(Exception $error){ + + header("X-Error: " . $error->getMessage()); +} diff --git a/banner/aves.png b/banner/aves.png Binary files differnew file mode 100644 index 0000000..ace604f --- /dev/null +++ b/banner/aves.png diff --git a/banner/aves_2.png b/banner/aves_2.png Binary files differnew file mode 100644 index 0000000..c78839f --- /dev/null +++ b/banner/aves_2.png diff --git a/banner/bibblebop.png b/banner/bibblebop.png Binary files differnew file mode 100644 index 0000000..0c061e0 --- /dev/null +++ b/banner/bibblebop.png diff --git a/banner/birds birds birdsw_4.jpg b/banner/birds birds birdsw_4.jpg Binary files differnew file mode 100644 index 0000000..ba7d637 --- /dev/null +++ b/banner/birds birds birdsw_4.jpg diff --git a/banner/birds_birds_birdsw.jpg b/banner/birds_birds_birdsw.jpg Binary files differnew file mode 100644 index 0000000..ff04b23 --- /dev/null +++ b/banner/birds_birds_birdsw.jpg diff --git a/banner/birds_birds_birdsw_2.jpg b/banner/birds_birds_birdsw_2.jpg Binary files differnew file mode 100644 index 0000000..dcd6125 --- /dev/null +++ b/banner/birds_birds_birdsw_2.jpg diff --git a/banner/birds_birds_birdsw_3.jpg b/banner/birds_birds_birdsw_3.jpg Binary files differnew file mode 100644 index 0000000..1446207 --- /dev/null +++ b/banner/birds_birds_birdsw_3.jpg diff --git a/banner/deek.png b/banner/deek.png Binary files differnew file mode 100644 index 0000000..ef80354 --- /dev/null +++ b/banner/deek.png diff --git a/banner/deekchat.gif b/banner/deekchat.gif Binary files differnew file mode 100644 index 0000000..bba01da --- /dev/null +++ b/banner/deekchat.gif diff --git a/banner/eagle.png b/banner/eagle.png Binary files differnew file mode 100644 index 0000000..f074341 --- /dev/null +++ b/banner/eagle.png diff --git a/banner/eagle2.png b/banner/eagle2.png Binary files differnew file mode 100644 index 0000000..175366b --- /dev/null +++ b/banner/eagle2.png diff --git a/banner/eagle3.jpg b/banner/eagle3.jpg Binary files differnew file mode 100644 index 0000000..1e65b59 --- /dev/null +++ b/banner/eagle3.jpg diff --git a/banner/eddd_1.png b/banner/eddd_1.png Binary files differnew file mode 100644 index 0000000..fab460b --- /dev/null +++ b/banner/eddd_1.png diff --git a/banner/eddd_2.png b/banner/eddd_2.png Binary files differnew file mode 100644 index 0000000..5ce4c2c --- /dev/null +++ b/banner/eddd_2.png diff --git a/banner/eddd_3.png b/banner/eddd_3.png Binary files differnew file mode 100644 index 0000000..b4ca48d --- /dev/null +++ b/banner/eddd_3.png diff --git a/banner/gnuwu.png b/banner/gnuwu.png Binary files differnew file mode 100644 index 0000000..634b59d --- /dev/null +++ b/banner/gnuwu.png diff --git a/banner/gnuwu_2.png b/banner/gnuwu_2.png Binary files differnew file mode 100644 index 0000000..493a6d9 --- /dev/null +++ b/banner/gnuwu_2.png diff --git a/banner/horse.png b/banner/horse.png Binary files differnew file mode 100644 index 0000000..0075a9c --- /dev/null +++ b/banner/horse.png diff --git a/banner/linucks.jpg b/banner/linucks.jpg Binary files differnew file mode 100644 index 0000000..8874451 --- /dev/null +++ b/banner/linucks.jpg diff --git a/banner/real_nig_3.jpg b/banner/real_nig_3.jpg Binary files differnew file mode 100644 index 0000000..8091146 --- /dev/null +++ b/banner/real_nig_3.jpg diff --git a/banner/sec.png b/banner/sec.png Binary files differnew file mode 100644 index 0000000..3c1a49e --- /dev/null +++ b/banner/sec.png diff --git a/banner/tagmachine.png b/banner/tagmachine.png Binary files differnew file mode 100644 index 0000000..c8b82a0 --- /dev/null +++ b/banner/tagmachine.png diff --git a/favicon.ico b/favicon.ico Binary files differnew file mode 100644 index 0000000..e5c1fbc --- /dev/null +++ b/favicon.ico diff --git a/favicon.php b/favicon.php new file mode 100644 index 0000000..dadb923 --- /dev/null +++ b/favicon.php @@ -0,0 +1,362 @@ +<?php + +if(!isset($_GET["s"])){ + + header("X-Error: Missing parameter (s)ite"); + die(); +} + +new favicon($_GET["s"]); + +class favicon{ + + public function __construct($url){ + + header("Content-Type: image/png"); + + if(substr_count($url, "/") !== 2){ + + header("X-Error: Only provide the protocol and domain"); + $this->defaulticon(); + } + + $filename = str_replace(["https://", "http://"], "", $url); + header("Content-Disposition: inline; filename=\"{$filename}.png\""); + + include "lib/curlproxy.php"; + $this->proxy = new proxy(false); + + $this->filename = parse_url($url, PHP_URL_HOST); + + /* + Check if we have the favicon stored locally + */ + if(file_exists("icons/" . $filename . ".png")){ + + $handle = fopen("icons/" . $filename . ".png", "r"); + echo fread($handle, filesize("icons/" . $filename . ".png")); + fclose($handle); + return; + } + + /* + Scrape html + */ + try{ + + $payload = $this->proxy->get($url, $this->proxy::req_web, true); + + }catch(Exception $error){ + + header("X-Error: Could not fetch HTML (" . $error->getMessage() . ")"); + $this->favicon404(); + } + //$payload["body"] = '<link rel="manifest" id="MANIFEST_LINK" href="/data/manifest/" crossorigin="use-credentials" />'; + + // get link tags + preg_match_all( + '/< *link +(.*)[\/]?>/Uixs', + $payload["body"], + $linktags + ); + + /* + Get relevant tags + */ + + $linktags = $linktags[1]; + $attributes = []; + + /* + header("Content-Type: text/plain"); + print_r($linktags); + print_r($payload); + die();*/ + + for($i=0; $i<count($linktags); $i++){ + + // get attributes + preg_match_all( + '/([A-Za-z0-9]+) *= *("[^"]*"|[^" ]+)/s', + $linktags[$i], + $tags + ); + + for($k=0; $k<count($tags[1]); $k++){ + + $attributes[$i][] = [ + "name" => $tags[1][$k], + "value" => trim($tags[2][$k], "\" \n\r\t\v\x00") + ]; + } + } + + unset($payload); + unset($linktags); + + $href = []; + + // filter out the tags we want + foreach($attributes as &$group){ + + $tmp_href = null; + $tmp_rel = null; + $badtype = false; + + foreach($group as &$attribute){ + + switch($attribute["name"]){ + + case "rel": + + $attribute["value"] = strtolower($attribute["value"]); + + if( + ( + $attribute["value"] == "icon" || + $attribute["value"] == "manifest" || + $attribute["value"] == "shortcut icon" || + $attribute["value"] == "apple-touch-icon" || + $attribute["value"] == "mask-icon" + ) === false + ){ + + break; + } + + $tmp_rel = $attribute["value"]; + break; + + case "type": + $attribute["value"] = explode("/", $attribute["value"], 2); + + if(strtolower($attribute["value"][0]) != "image"){ + + $badtype = true; + break; + } + break; + + case "href": + + // must not contain invalid characters + // must be bigger than 1 + if( + filter_var($attribute["value"], FILTER_SANITIZE_URL) == $attribute["value"] && + strlen($attribute["value"]) > 0 + ){ + + $tmp_href = $attribute["value"]; + break; + } + break; + } + } + + if( + $badtype === false && + $tmp_rel !== null && + $tmp_href !== null + ){ + + $href[$tmp_rel] = $tmp_href; + } + } + + /* + Priority list + */ + /* + header("Content-Type: text/plain"); + print_r($href); + die();*/ + + if(isset($href["icon"])){ $href = $href["icon"]; } + elseif(isset($href["apple-touch-icon"])){ $href = $href["apple-touch-icon"]; } + elseif(isset($href["manifest"])){ + + // attempt to parse manifest, but fallback to [] + $href = $this->parsemanifest($href["manifest"], $url); + } + + if(is_array($href)){ + + if(isset($href["mask-icon"])){ $href = $href["mask-icon"]; } + elseif(isset($href["shortcut icon"])){ $href = $href["shortcut icon"]; } + else{ + + $href = "/favicon.ico"; + } + } + + $href = $this->proxy->getabsoluteurl($href, $url); + /* + header("Content-type: text/plain"); + echo $href; + die();*/ + + + /* + Download the favicon + */ + //$href = "https://git.lolcat.ca/assets/img/logo.svg"; + + try{ + $payload = + $this->proxy->get( + $href, + $this->proxy::req_image, + true, + $url + ); + + }catch(Exception $error){ + + header("X-Error: Could not fetch the favicon (" . $error->getMessage() . ")"); + $this->favicon404(); + } + + /* + Parse the file format + */ + $image = null; + $format = $this->proxy->getimageformat($payload, $image); + + /* + Convert the image + */ + try{ + + /* + @todo: fix issues with avif+transparency + maybe using GD as fallback? + */ + if($format !== false){ + $image->setFormat($format); + } + + $image->setBackgroundColor(new ImagickPixel("transparent")); + $image->readImageBlob($payload["body"]); + $image->resizeImage(16, 16, imagick::FILTER_LANCZOS, 1); + $image->setFormat("png"); + + $image = $image->getImageBlob(); + + // save favicon + $handle = fopen("icons/" . $this->filename . ".png", "w"); + fwrite($handle, $image, strlen($image)); + fclose($handle); + + echo $image; + + }catch(ImagickException $error){ + + header("X-Error: Could not convert the favicon: (" . $error->getMessage() . ")"); + $this->favicon404(); + } + + return; + } + + private function parsemanifest($href, $url){ + + if( + // check if base64-encoded JSON manifest + preg_match( + '/^data:application\/json;base64,([A-Za-z0-9=]*)$/', + $href, + $json + ) + ){ + + $json = base64_decode($json[1]); + + if($json === false){ + + // could not decode the manifest regex + return []; + } + + }else{ + + try{ + $json = + $this->proxy->get( + $this->proxy->getabsoluteurl($href, $url), + $this->proxy::req_web, + false, + $url + ); + + $json = $json["body"]; + + }catch(Exception $error){ + + // could not fetch the manifest + return []; + } + } + + $json = json_decode($json, true); + + if($json === null){ + + // manifest did not return valid json + return []; + } + + if( + isset($json["start_url"]) && + $this->proxy->validateurl($json["start_url"]) + ){ + + $url = $json["start_url"]; + } + + if(!isset($json["icons"][0]["src"])){ + + // manifest does not contain a path to the favicon + return []; + } + + // horay, return the favicon path + return $json["icons"][0]["src"]; + } + + private function favicon404(){ + + // fallback to google favicons + // ... probably blocked by cuckflare + try{ + + $image = + $this->proxy->get( + "https://t0.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url=http://{$this->filename}&size=16", + $this->proxy::req_image + ); + }catch(Exception $error){ + + $this->defaulticon(); + } + + // write favicon from google + $handle = fopen("icons/" . $this->filename . ".png", "w"); + fwrite($handle, $image["body"], strlen($image["body"])); + fclose($handle); + + echo $image["body"]; + die(); + } + + private function defaulticon(){ + + // give 404 and fuck off + http_response_code(404); + + $handle = fopen("lib/favicon404.png", "r"); + echo fread($handle, filesize("lib/favicon404.png")); + fclose($handle); + + die(); + } +} diff --git a/icons/lolcat.ca.png b/icons/lolcat.ca.png Binary files differnew file mode 100644 index 0000000..c7e4785 --- /dev/null +++ b/icons/lolcat.ca.png diff --git a/images.php b/images.php new file mode 100644 index 0000000..67a50e8 --- /dev/null +++ b/images.php @@ -0,0 +1,99 @@ +<?php + +/* + Initialize random shit +*/ +include "lib/frontend.php"; +$frontend = new frontend(); + +[$scraper, $filters] = $frontend->getscraperfilters("images"); + +$get = $frontend->parsegetfilters($_GET, $filters); + +$frontend->loadheader( + $get, + $filters, + "images" +); + +$payload = [ + "images" => "", + "nextpage" => "" +]; + +try{ + $results = $scraper->image($get); + +}catch(Exception $error){ + + echo + $frontend->drawerror( + "Shit", + 'This scraper returned an error:' . + '<div class="code">' . htmlspecialchars($error->getMessage()) . '</div>' . + 'Things you can try:' . + '<ul>' . + '<li>Use a different scraper</li>' . + '<li>Remove keywords that could cause errors</li>' . + '<li>Use another 4get instance</li>' . + '</ul><br>' . + 'If the error persists, please <a href="/about">contact the administrator</a>.' + ); + die(); +} + +if(count($results["image"]) === 0){ + + $payload["images"] = + '<div class="infobox">' . + "<h1>Nobody here but us chickens!</h1>" . + 'Have you tried:' . + '<ul>' . + '<li>Using a different scraper</li>' . + '<li>Using fewer keywords</li>' . + '<li>Defining broader filters (Is NSFW turned off?)</li>' . + '</ul>' . + '</div>'; +} + +foreach($results["image"] as $image){ + + $domain = htmlspecialchars(parse_url($image["url"], PHP_URL_HOST)); + + $c = count($image["source"]) - 1; + + if( + preg_match( + '/^data:/', + $image["source"][$c]["url"] + ) + ){ + + $src = htmlspecialchars($image["source"][$c]["url"]); + }else{ + + $src = "/proxy?i=" . urlencode($image["source"][$c]["url"]) . "&s=thumb"; + } + + $payload["images"] .= + '<div class="image-wrapper" title="' . htmlspecialchars($image["title"]) .'" data-json="' . htmlspecialchars(json_encode($image["source"])) . '">' . + '<div class="image">' . + '<a href="' . htmlspecialchars($image["source"][0]["url"]) . '" rel="noreferrer nofollow" class="thumb">' . + '<img src="' . $src . '" alt="thumbnail">' . + '<div class="duration">' . $image["source"][0]["width"] . 'x' . $image["source"][0]["height"] . '</div>' . + '</a>' . + '<a href="' . htmlspecialchars($image["url"]) . '" rel="noreferrer nofollow">' . + '<div class="title">' . htmlspecialchars($domain) . '</div>' . + '<div class="description">' . $frontend->highlighttext($get["s"], $image["title"]) . '</div>' . + '</a>' . + '</div>' . + '</div>'; +} + +if($results["npt"] !== null){ + + $payload["nextpage"] = + '<a href="' . $frontend->htmlnextpage($get, $results["npt"], "images") . '" class="nextpage img">Next page ></a>'; +} + +echo $frontend->load("images.html", $payload); diff --git a/index.php b/index.php new file mode 100644 index 0000000..be9897f --- /dev/null +++ b/index.php @@ -0,0 +1,14 @@ +<?php + +include "lib/frontend.php"; +$frontend = new frontend(); + +$images = glob("banner/*"); + +echo $frontend->load( + "home.html", + [ + "body_class" => $frontend->getthemeclass(false), + "banner" => $images[rand(0, count($images) - 1)] + ] +); diff --git a/lib/bingcache-todo-fix.php b/lib/bingcache-todo-fix.php new file mode 100644 index 0000000..a4acb5b --- /dev/null +++ b/lib/bingcache-todo-fix.php @@ -0,0 +1,144 @@ +<?php + +// https://www.bing.com/search?q=url%3Ahttps%3A%2F%2Flolcat.ca +// https://cc.bingj.com/cache.aspx?q=url%3ahttps%3a%2f%2flolcat.ca&d=4769685974291356&mkt=en-CA&setlang=en-US&w=tEsWuE7HW3Z5AIPQMVkDH4WaotS4LrK- +// <div class="b_attribution" u="0N|5119|4769685974291356|tEsWuE7HW3Z5AIPQMVkDH4WaotS4LrK-" tabindex="0"> + +new bingcache(); + +class bingcache{ + + public function __construct(){ + + if( + !isset($_GET["s"]) || + $this->validate_url($_GET["s"]) === false + ){ + + var_dump($this->validate_url($_GET["s"])); + $this->do404("Please provide a valid URL."); + } + + $url = $_GET["s"]; + + $curlproc = curl_init(); + + curl_setopt( + $curlproc, + CURLOPT_URL, + "https://www.bing.com/search?q=url%3A" . + urlencode($url) + ); + + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + curl_setopt( + $curlproc, + CURLOPT_HTTPHEADER, + ["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/107.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip", + "DNT: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1"] + ); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 5); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + $this->do404("Failed to connect to bing servers. Please try again later."); + } + + curl_close($curlproc); + + preg_match( + '/<div class="b_attribution" u="(.*)" tabindex="0">/', + $data, + $keys + ); + + print_r($keys); + + if(count($keys) === 0){ + + $this->do404("Bing has not archived this URL."); + } + + $keys = explode("|", $keys[1]); + $count = count($keys); + + //header("Location: https://cc.bingj.com/cache.aspx?d=" . $keys[$count - 2] . "&w=" . $keys[$count - 1]); + echo("Location: https://cc.bingj.com/cache.aspx?d=" . $keys[$count - 2] . "&w=" . $keys[$count - 1]); + } + + public function do404($text){ + + include "lib/frontend.php"; + $frontend = new frontend(); + + echo + $frontend->load( + "error.html", + [ + "title" => "Shit", + "text" => $text + ] + ); + + die(); + } + + public function validate_url($url){ + + $url_parts = parse_url($url); + + // check if required parts are there + if( + !isset($url_parts["scheme"]) || + !( + $url_parts["scheme"] == "http" || + $url_parts["scheme"] == "https" + ) || + !isset($url_parts["host"]) + ){ + return false; + } + + if( + // if its not an RFC-valid URL + !filter_var($url, FILTER_VALIDATE_URL) + ){ + return false; + } + + $ip = + str_replace( + ["[", "]"], // handle ipv6 + "", + $url_parts["host"] + ); + + // if its not an IP + if(!filter_var($ip, FILTER_VALIDATE_IP)){ + + // resolve domain's IP + $ip = gethostbyname($url_parts["host"] . "."); + } + + // check if its localhost + return filter_var( + $ip, + FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE + ); + } +} diff --git a/lib/classic.png b/lib/classic.png Binary files differnew file mode 100644 index 0000000..3d2b8fc --- /dev/null +++ b/lib/classic.png diff --git a/lib/curlproxy.php b/lib/curlproxy.php new file mode 100644 index 0000000..846fbb7 --- /dev/null +++ b/lib/curlproxy.php @@ -0,0 +1,652 @@ +<?php + +class proxy{ + + public const req_web = 0; + public const req_image = 1; + + public function __construct($cache = true){ + + $this->cache = $cache; + } + + public function do404(){ + + http_response_code(404); + header("Content-Type: image/png"); + + $handle = fopen("lib/img404.png", "r"); + echo fread($handle, filesize("lib/img404.png")); + fclose($handle); + + die(); + return; + } + + public function getabsoluteurl($path, $relative){ + + if($this->validateurl($path)){ + + return $path; + } + + if(substr($path, 0, 2) == "//"){ + + return "https:" . $path; + } + + $url = null; + + $relative = parse_url($relative); + $url = $relative["scheme"] . "://"; + + if( + isset($relative["user"]) && + isset($relative["pass"]) + ){ + + $url .= $relative["user"] . ":" . $relative["pass"] . "@"; + } + + $url .= $relative["host"]; + + if(isset($relative["path"])){ + + $relative["path"] = explode( + "/", + $relative["path"] + ); + + unset($relative["path"][count($relative["path"]) - 1]); + $relative["path"] = implode("/", $relative["path"]); + + $url .= $relative["path"]; + } + + if( + strlen($path) !== 0 && + $path[0] !== "/" + ){ + + $url .= "/"; + } + + $url .= $path; + + return $url; + } + + public function validateurl($url){ + + $url_parts = parse_url($url); + + // check if required parts are there + if( + !isset($url_parts["scheme"]) || + !( + $url_parts["scheme"] == "http" || + $url_parts["scheme"] == "https" + ) || + !isset($url_parts["host"]) + ){ + return false; + } + + $ip = + str_replace( + ["[", "]"], // handle ipv6 + "", + $url_parts["host"] + ); + + // if its not an IP + if(!filter_var($ip, FILTER_VALIDATE_IP)){ + + // resolve domain's IP + $ip = gethostbyname($url_parts["host"] . "."); + } + + // check if its localhost + if( + filter_var( + $ip, + FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE + ) === false + ){ + + return false; + } + + return true; + } + + public function get($url, $reqtype = self::req_web, $acceptallcodes = false, $referer = null, $redirectcount = 0){ + + if($redirectcount === 5){ + + throw new Exception("Too many redirects"); + } + + // sanitize URL + try{ + + $this->validateurl($url); + }catch(Exception $error){ + + throw new Exception($error->getMessage()); + } + + $this->clientcache(); + + $curl = curl_init(); + + curl_setopt($curl, CURLOPT_URL, $url); + curl_setopt($curl, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curl, CURLOPT_HEADER, 1); + + switch($reqtype){ + case self::req_web: + curl_setopt( + $curl, + CURLOPT_HTTPHEADER, + [ + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate", + "DNT: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1" + ] + ); + break; + + case self::req_image: + + if($referer === null){ + $referer = explode("/", $url, 4); + array_pop($referer); + + $referer = implode("/", $referer); + } + + curl_setopt( + $curl, + CURLOPT_HTTPHEADER, + [ + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/107.0", + "Accept: image/avif,image/webp,*/*", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate", + "DNT: 1", + "Connection: keep-alive", + "Referer: {$referer}" + ] + ); + break; + } + + curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curl, CURLOPT_TIMEOUT, 30); + + // limit size of payloads + curl_setopt($curl, CURLOPT_BUFFERSIZE, 1024); + curl_setopt($curl, CURLOPT_NOPROGRESS, false); + curl_setopt( + $curl, + CURLOPT_PROGRESSFUNCTION, + function($downloadsize, $downloaded, $uploadsize, $uploaded + ){ + + // if $downloaded exceeds 100MB, fuck off + return ($downloaded > 100000000) ? 1 : 0; + }); + + $body = curl_exec($curl); + + if(curl_errno($curl)){ + + throw new Exception(curl_error($curl)); + } + + curl_close($curl); + + $headers = []; + $http = null; + + while(true){ + + $header = explode("\n", $body, 2); + $body = $header[1]; + + if($http === null){ + + // http/1.1 200 ok + $header = explode("/", $header[0], 2); + $header = explode(" ", $header[1], 3); + + $http = [ + "version" => (float)$header[0], + "code" => (int)$header[1] + ]; + + continue; + } + + if(trim($header[0]) == ""){ + + // reached end of headers + break; + } + + $header = explode(":", $header[0], 2); + + // malformed headers + if(count($header) !== 2){ continue; } + + $headers[strtolower(trim($header[0]))] = trim($header[1]); + } + + // check http code + if( + $http["code"] >= 300 && + $http["code"] <= 309 + ){ + + // redirect + if(!isset($headers["location"])){ + + throw new Exception("Broken redirect"); + } + + $redirectcount++; + + return $this->get($this->getabsoluteurl($headers["location"], $url), $reqtype, $acceptallcodes, $referer, $redirectcount); + }else{ + if( + $acceptallcodes === false && + $http["code"] > 300 + ){ + + throw new Exception("Remote server returned an error code! ({$http["code"]})"); + } + } + + // check if data is okay + switch($reqtype){ + + case self::req_image: + + $format = false; + + if(isset($headers["content-type"])){ + + if($headers["content-type"] == "text/html"){ + + throw new Exception("Server returned an html document instead of image"); + } + + $tmp = explode(";", $headers["content-type"]); + + for($i=0; $i<count($tmp); $i++){ + + if( + preg_match( + '/^image\/([^ ]+)/i', + $tmp[$i], + $match + ) + ){ + + $format = strtolower($match[1]); + + if(substr($format, 0, 2) == "x-"){ + + $format = substr($format, 2); + } + break; + } + } + } + + return [ + "http" => $http, + "format" => $format, + "headers" => $headers, + "body" => $body + ]; + break; + + default: + + return [ + "http" => $http, + "headers" => $headers, + "body" => $body + ]; + break; + } + + return; + } + + public function stream_linear_image($url, $referer = null){ + + $this->stream($url, $referer, "image"); + } + + public function stream_linear_audio($url, $referer = null){ + + $this->stream($url, $referer, "audio"); + } + + private function stream($url, $referer, $format){ + + $this->url = $url; + $this->format = $format; + + // sanitize URL + try{ + + $this->validateurl($url); + }catch(Exception $error){ + + throw new Exception($error->getMessage()); + } + + $this->clientcache(); + + $curl = curl_init(); + + // set headers + if($referer === null){ + $referer = explode("/", $url, 4); + array_pop($referer); + + $referer = implode("/", $referer); + } + + switch($format){ + + case "image": + curl_setopt( + $curl, + CURLOPT_HTTPHEADER, + [ + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0", + "Accept: image/avif,image/webp,*/*", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate, br", + "DNT: 1", + "Connection: keep-alive", + "Referer: {$referer}" + ] + ); + break; + + case "audio": + curl_setopt( + $curl, + CURLOPT_HTTPHEADER, + [ + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0", + "Accept: audio/webm,audio/ogg,audio/wav,audio/*;q=0.9,application/ogg;q=0.7,video/*;q=0.6,*/*;q=0.5", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate, br", + "DNT: 1", + "Connection: keep-alive", + "Referer: {$referer}" + ] + ); + break; + } + + // follow redirects + curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($curl, CURLOPT_MAXREDIRS, 5); + curl_setopt($curl, CURLOPT_AUTOREFERER, 5); + + // set url + curl_setopt($curl, CURLOPT_URL, $url); + curl_setopt($curl, CURLOPT_ENCODING, ""); // default encoding + + // timeout + disable ssl + curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, 10); + curl_setopt($curl, CURLOPT_TIMEOUT, 30); + + curl_setopt( + $curl, + CURLOPT_WRITEFUNCTION, + function($c, $data){ + + if(curl_getinfo($c, CURLINFO_HTTP_CODE) !== 200){ + + throw new Exception("Serber returned a non-200 code"); + } + + echo $data; + return strlen($data); + } + ); + + $this->empty_header = false; + $this->cont = false; + $this->headers_tmp = []; + $this->headers = []; + curl_setopt( + $curl, + CURLOPT_HEADERFUNCTION, + function($c, $header){ + + $head = trim($header); + $len = strlen($head); + + if($len === 0){ + + $this->empty_header = true; + $this->headers_tmp = []; + }else{ + + $this->empty_header = false; + $this->headers_tmp[] = $head; + } + + foreach($this->headers_tmp as $h){ + + // parse headers + $h = explode(":", $h, 2); + + if(count($h) !== 2){ + + if(curl_getinfo($c, CURLINFO_HTTP_CODE) !== 200){ + + // not HTTP 200, probably a redirect + $this->cont = false; + }else{ + + $this->cont = true; + } + + // is HTTP 200, just ignore that line + continue; + } + + $this->headers[strtolower(trim($h[0]))] = trim($h[1]); + } + + if( + $this->cont && + $this->empty_header + ){ + + // get content type + if(isset($this->headers["content-type"])){ + + $filetype = explode("/", $this->headers["content-type"]); + + if(strtolower($filetype[0]) != $this->format){ + + throw new Exception("Resource is not an {$this->format} (Found {$filetype[0]} instead)"); + } + + }else{ + + throw new Exception("Resource is not an {$this->format} (no Content-Type)"); + } + + header("Content-Type: {$this->format}/{$filetype[1]}"); + + // give payload size + if(isset($this->headers["content-length"])){ + + header("Content-Length: {$this->headers["content-length"]}"); + } + + // give filename + $this->getfilenameheader($this->headers, $this->url, $filetype[1]); + } + + return strlen($header); + } + ); + + curl_exec($curl); + + if(curl_errno($curl)){ + + throw new Exception(curl_error($curl)); + } + + curl_close($curl); + } + + public function getfilenameheader($headers, $url, $filetype = "jpg"){ + + // get filename from content-disposition header + if(isset($headers["content-disposition"])){ + + preg_match( + '/filename=([^;]+)/', + $headers["content-disposition"], + $filename + ); + + if(isset($filename[1])){ + + header("Content-Disposition: filename=" . $filename[1] . "." . $filetype); + return; + } + } + + // get filename from URL + $filename = parse_url($url, PHP_URL_PATH); + + if($filename === null){ + + // everything failed! rename file to domain name + header("Content-Disposition: filename=" . parse_url($url, PHP_URL_HOST) . "." . $filetype); + return; + } + + // remove extension from filename + $filename = + explode( + ".", + basename($filename) + ); + + if(count($filename) > 1){ + array_pop($filename); + } + + $filename = implode(".", $filename); + + header("Content-Disposition: inline; filename=" . $filename . "." . $filetype); + return; + } + + public function getimageformat($payload, &$imagick){ + + $finfo = new finfo(FILEINFO_MIME_TYPE); + $format = $finfo->buffer($payload["body"]); + + if($format === false){ + + if($payload["format"] === false){ + + header("X-Error: Could not parse format"); + $this->favicon404(); + } + + $format = $payload["format"]; + }else{ + + $format_tmp = explode("/", $format, 2); + + if($format_tmp[0] == "image"){ + + $format_tmp = strtolower($format_tmp[1]); + + if(substr($format_tmp, 0, 2) == "x-"){ + + $format_tmp = substr($format_tmp, 2); + } + + $format = $format_tmp; + } + } + + switch($format){ + + case "tiff": $format = "gif"; break; + case "vnd.microsoft.icon": $format = "ico"; break; + case "icon": $format = "ico"; break; + case "svg+xml": $format = "svg"; break; + } + + $imagick = new Imagick(); + + if( + !in_array( + $format, + array_map("strtolower", $imagick->queryFormats()) + ) + ){ + + // format could not be found, but imagemagick can + // sometimes detect it? shit's fucked + $format = false; + } + + return $format; + } + + public function clientcache(){ + + if($this->cache === false){ + + return; + } + + header("Last-Modified: Thu, 01 Oct 1970 00:00:00 GMT"); + $headers = getallheaders(); + + if( + isset($headers["If-Modified-Since"]) || + isset($headers["If-Unmodified-Since"]) + ){ + + http_response_code(304); // 304: Not Modified + die(); + } + } +} diff --git a/lib/favicon404.png b/lib/favicon404.png Binary files differnew file mode 100644 index 0000000..7540694 --- /dev/null +++ b/lib/favicon404.png diff --git a/lib/frontend.php b/lib/frontend.php new file mode 100644 index 0000000..3be912b --- /dev/null +++ b/lib/frontend.php @@ -0,0 +1,1282 @@ +<?php + +class frontend{ + + public function load($template, $replacements = []){ + + $handle = fopen("template/{$template}", "r"); + $data = fread($handle, filesize("template/{$template}")); + fclose($handle); + + $data = explode("\n", $data); + $html = ""; + + for($i=0; $i<count($data); $i++){ + + $html .= trim($data[$i]); + } + + foreach($replacements as $key => $value){ + + $html = + str_replace( + "{%{$key}%}", + $value, + $html + ); + } + + return trim($html); + } + + public function getthemeclass($raw = true){ + + if( + isset($_COOKIE["theme"]) && + $_COOKIE["theme"] == "cream" + ){ + + $body_class = "theme-white "; + }else{ + + $body_class = ""; + } + + if( + $raw && + $body_class != "" + ){ + + return ' class="' . rtrim($body_class) . '"'; + } + + return $body_class; + } + + public function loadheader(array $get, array $filters, string $page){ + + echo + $this->load("header.html", [ + "title" => trim($get["s"] . " ({$page})"), + "description" => ucfirst($page) . ' search results for "' . htmlspecialchars($get["s"]) . '"', + "index" => "no", + "search" => htmlspecialchars($get["s"]), + "tabs" => $this->generatehtmltabs($page, $get["s"]), + "filters" => $this->generatehtmlfilters($filters, $get), + "body_class" => $this->getthemeclass() + ]); + + if( + preg_match( + '/bot|wget|curl|python-requests|scrapy|feedfetcher|go-http-client|ruby|universalfeedparser|yahoo\! slurp|spider|rss/i', + $_SERVER["HTTP_USER_AGENT"] + ) + ){ + + // bot detected !! + echo + $this->drawerror( + "Tshh, blocked!", + 'You were blocked from viewing this page. If you wish to scrape data from 4get, please consider running <a href="https://git.lolcat.ca/lolcat/4get" rel="noreferrer nofollow">your own 4get instance</a> or using <a href="/api.txt">the API</a>.', + ); + die(); + } + } + + public function drawerror($title, $error){ + + return + $this->load("search.html", [ + "class" => "", + "right-left" => "", + "right-right" => "", + "left" => + '<div class="infobox">' . + '<h1>' . htmlspecialchars($title) . '</h1>' . + $error . + '</div>' + ]); + } + + public function drawtextresult($site, $greentext = null, $duration = null, $keywords, $tabindex = true){ + + $payload = + '<div class="text-result">'; + + // add favicon, link and archive links + $payload .= $this->drawlink($site["url"]); + + /* + Draw title + description + filetype + */ + $payload .= + '<a href="' . htmlspecialchars($site["url"]) . '" class="hover" rel="noreferrer nofollow"'; + + if($tabindex === false){ + + $payload .= ' tabindex="-1"'; + } + + $payload .= '>'; + + if($site["thumb"]["url"] !== null){ + + $payload .= + '<div class="thumb-wrap'; + + switch($site["thumb"]["ratio"]){ + + case "16:9": + $size = "landscape"; + break; + + case "9:16": + $payload .= " portrait"; + $size = "portrait"; + break; + + case "1:1": + $payload .= " square"; + $size = "square"; + break; + } + + $payload .= + '">' . + '<img class="thumb" src="/proxy?i=' . urlencode($site["thumb"]["url"]) . '&s=' . $size . '" alt="thumb">'; + + if($duration !== null){ + + $payload .= + '<div class="duration">' . + htmlspecialchars($duration) . + '</div>'; + } + + $payload .= + '</div>'; + } + + $payload .= + '<div class="title">'; + + if( + isset($site["type"]) && + $site["type"] != "web" + ){ + + $payload .= '<div class="type">' . strtoupper($site["type"]) . '</div>'; + } + + $payload .= + htmlspecialchars($site["title"]) . + '</div>'; + + if($greentext !== null){ + + $payload .= + '<div class="greentext">' . + htmlspecialchars($greentext) . + '</div>'; + } + + if($site["description"] !== null){ + + $payload .= + '<div class="description">' . + $this->highlighttext($keywords, $site["description"]) . + '</div>'; + } + + $payload .= '</a>'; + + /* + Sublinks + */ + if( + isset($site["sublink"]) && + !empty($site["sublink"]) + ){ + + usort($site["sublink"], function($a, $b){ + + return strlen($a["description"]) > strlen($b["description"]); + }); + + $payload .= + '<div class="sublinks">' . + '<table>'; + + $opentr = false; + for($i=0; $i<count($site["sublink"]); $i++){ + + if(($i % 2) === 0){ + + $opentr = true; + $payload .= '<tr>'; + }else{ + + $opentr = false; + } + + $payload .= + '<td>' . + '<a href="' . htmlspecialchars($site["sublink"][$i]["url"]) . '" rel="noreferrer nofollow">' . + '<div class="title">' . + htmlspecialchars($site["sublink"][$i]["title"]) . + '</div>'; + + if(!empty($site["sublink"][$i]["date"])){ + + $payload .= + '<div class="greentext">' . + date("jS M y @ g:ia", $site["sublink"][$i]["date"]) . + '</div>'; + } + + if(!empty($site["sublink"][$i]["description"])){ + + $payload .= + '<div class="description">' . + $this->highlighttext($keywords, $site["sublink"][$i]["description"]) . + '</div>'; + } + + $payload .= '</a></td>'; + + if($opentr === false){ + + $payload .= '</tr>'; + } + } + + if($opentr === true){ + + $payload .= '<td></td></tr>'; + } + + $payload .= '</table></div>'; + } + + if( + isset($site["table"]) && + !empty($site["table"]) + ){ + + $payload .= '<table class="info-table">'; + + foreach($site["table"] as $title => $value){ + + $payload .= + '<tr>' . + '<td>' . htmlspecialchars($title) . '</td>' . + '<td>' . htmlspecialchars($value) . '</td>' . + '</tr>'; + } + + $payload .= '</table>'; + } + + return $payload . '</div>'; + } + + public function highlighttext($keywords, $text){ + + $text = htmlspecialchars($text); + + $keywords = explode(" ", $keywords); + $regex = []; + + foreach($keywords as $word){ + + $regex[] = "\b" . preg_quote($word, "/") . "\b"; + } + + $regex = "/" . implode("|", $regex) . "/i"; + + return + preg_replace( + $regex, + '<b>${0}</b>', + $text + ); + } + + function highlightcode($text){ + + // https://www.php.net/highlight_string + ini_set("highlight.comment", "c-comment"); + ini_set("highlight.default", "c-default"); + ini_set("highlight.html", "c-default"); + ini_set("highlight.keyword", "c-keyword"); + ini_set("highlight.string", "c-string"); + + $text = + trim( + preg_replace( + '/<\/span>$/', + "", // remove stray ending span because of the <?php stuff + str_replace( + [ + '<br />', + ' ' + ], + [ + "\n", // replace <br> with newlines + " " // replace html entity to space + ], + str_replace( + [ + // leading <?php garbage + "<span style=\"color: c-default\">\n<?php ", + "<code>", + "</code>" + ], + "", + highlight_string("<?php " . $text, true) + ) + ) + ) + ); + + // replace colors + $classes = ["c-comment", "c-default", "c-keyword", "c-string"]; + + foreach($classes as $class){ + + $text = str_replace('<span style="color: ' . $class . '">', '<span class="' . $class . '">', $text); + } + + return $text; + } + + public function drawlink($link){ + + /* + Add favicon + */ + $host = parse_url($link); + $esc = + explode( + ".", + $host["host"], + 2 + ); + + if( + count($esc) === 2 && + $esc[0] == "www" + ){ + + $esc = $esc[1]; + }else{ + + $esc = $esc[0]; + } + + $esc = substr($esc, 0, 2); + + $urlencode = urlencode($link); + + $payload = + '<div class="url">' . + '<button class="favicon" tabindex="-1">' . + '<img src="/favicon?s=' . htmlspecialchars($host["scheme"] . "://" . $host["host"]) . '" alt="' . htmlspecialchars($esc) . '">' . + //'<img src="/404.php" alt="' . htmlspecialchars($esc) . '">' . + '</button>' . + '<div class="favicon-dropdown">'; + + /* + Add archive links + */ + if( + $host["host"] == "boards.4chan.org" || + $host["host"] == "boards.4channel.org" + ){ + + $archives = []; + $path = explode("/", $host["path"]); + $count = count($path); + // /pol/thread/417568063/post-shitty-memes-if-you-want-to + + if($count !== 0){ + + $isboard = true; + + switch($path[1]){ + + case "con": + break; + + case "q": + $archives[] = "desuarchive.org"; + break; + + case "qa": + $archives[] = "desuarchive.org"; + break; + + case "qb": + $archives[] = "arch.b4k.co"; + break; + + case "trash": + $archives[] = "desuarchive.org"; + break; + + case "a": + $archives[] = "desuarchive.org"; + break; + + case "c": + $archives[] = "desuarchive.org"; + break; + + case "w": + break; + + case "m": + $archives[] = "desuarchive.org"; + break; + + case "cgl": + $archives[] = "desuarchive.org"; + $archives[] = "warosu.org"; + break; + + case "cm": + $archives[] = "boards.fireden.net"; + break; + + case "f": + $archives[] = "archive.4plebs.org"; + break; + + case "n": + break; + + case "jp": + $archives[] = "warosu.org"; + break; + + case "vt": + $archives[] = "warosu.org"; + break; + + case "v": + $archives[] = "boards.fireden.net"; + $archives[] = "arch.b4k.co"; + break; + + case "vg": + $archives[] = "boards.fireden.net"; + $archives[] = "arch.b4k.co"; + break; + + case "vm": + $archives[] = "arch.b4k.co"; + break; + + case "vmg": + $archives[] = "arch.b4k.co"; + break; + + case "vp": + $archives[] = "arch.b4k.co"; + break; + + case "vr": + $archives[] = "desuarchive.org"; + $archives[] = "warosu.org"; + break; + + case "vrpg": + $archives[] = "arch.b4k.co"; + break; + + case "vst": + $archives[] = "arch.b4k.co"; + break; + + case "co": + $archives[] = "desuarchive.org"; + break; + + case "g": + $archives[] = "desuarchive.org"; + $archives[] = "arch.b4k.co"; + break; + + case "tv": + $archives[] = "archive.4plebs.org"; + break; + + case "k": + $archives[] = "desuarchive.org"; + break; + + case "o": + $archives[] = "archive.4plebs.org"; + break; + + case "an": + $archives[] = "desuarchive.org"; + break; + + case "tg": + $archives[] = "desuarchive.org"; + $archives[] = "archive.4plebs.org"; + break; + + case "sp": + $archives[] = "archive.4plebs.org"; + break; + + case "xs": + $archives[] = "eientei.xyz"; + break; + + case "pw": + break; + + case "sci": + $archives[] = "boards.fireden.net"; + $archives[] = "warosu.org"; + $archives[] = "eientei.xyz"; + break; + + case "his": + $archives[] = "desuarchive.org"; + break; + + case "int": + $archives[] = "desuarchive.org"; + break; + + case "out": + break; + + case "toy": + break; + + case "i": + $archives[] = "archiveofsins.com"; + $archives[] = "eientei.xyz"; + break; + + case "po": + break; + + case "p": + break; + + case "ck": + $archives[] = "warosu.org"; + break; + + case "ic": + $archives[] = "boards.fireden.net"; + $archives[] = "warosu.org"; + break; + + case "wg": + break; + + case "lit": + $archives[] = "warosu.org"; + break; + + case "mu": + $archives[] = "desuarchive.org"; + break; + + case "fa": + $archives[] = "warosu.org"; + break; + + case "3": + $archives[] = "warosu.org"; + $archives[] = "eientei.xyz"; + break; + + case "gd": + break; + + case "diy": + $archives[] = "warosu.org"; + break; + + case "wsg": + $archives[] = "desuarchive.org"; + break; + + case "qst": + break; + + case "biz": + $archives[] = "warosu.org"; + break; + + case "trv": + $archives[] = "archive.4plebs.org"; + break; + + case "fit": + $archives[] = "desuarchive.org"; + break; + + case "x": + $archives[] = "archive.4plebs.org"; + break; + + case "adv": + $archives[] = "archive.4plebs.org"; + break; + + case "lgbt": + $archives[] = "archiveofsins.com"; + break; + + case "mlp": + $archives[] = "desuarchive.org"; + $archives[] = "arch.b4k.co"; + break; + + case "news": + break; + + case "wsr": + break; + + case "vip": + break; + + case "b": + $archives[] = "thebarchive.com"; + break; + + case "r9k": + $archives[] = "desuarchive.org"; + break; + + case "pol": + $archives[] = "archive.4plebs.org"; + break; + + case "bant": + $archives[] = "thebarchive.com"; + break; + + case "soc": + $archives[] = "archiveofsins.com"; + break; + + case "s4s": + $archives[] = "archive.4plebs.org"; + break; + + case "s": + $archives[] = "archiveofsins.com"; + break; + + case "hc": + $archives[] = "archiveofsins.com"; + break; + + case "hm": + $archives[] = "archiveofsins.com"; + break; + + case "h": + $archives[] = "archiveofsins.com"; + break; + + case "e": + break; + + case "u": + $archives[] = "archiveofsins.com"; + break; + + case "d": + $archives[] = "desuarchive.org"; + break; + + case "y": + $archives[] = "boards.fireden.net"; + break; + + case "t": + $archives[] = "archiveofsins.com"; + break; + + case "hr": + $archives[] = "archive.4plebs.org"; + break; + + case "gif": + break; + + case "aco": + $archives[] = "desuarchive.org"; + break; + + case "r": + $archives[] = "archiveofsins.com"; + break; + + default: + $isboard = false; + break; + } + + if($isboard === true){ + + $archives[] = "archived.moe"; + } + + $trail = ""; + + if( + isset($path[2]) && + isset($path[3]) && + $path[2] == "thread" + ){ + + $trail .= "/" . $path[1] . "/thread/" . $path[3]; + }elseif($isboard){ + + $trail = "/" . $path[1] . "/"; + } + + for($i=0; $i<count($archives); $i++){ + + $payload .= + '<a href="https://' . $archives[$i] . $trail . '" class="list" target="_BLANK">' . + '<img src="/favicon?s=https://' . $archives[$i] . '" alt="' . $archives[$i][0] . $archives[$i][1] . '">' . + $archives[$i] . + '</a>'; + } + } + } + + $payload .= + '<a href="https://webcache.googleusercontent.com/search?q=cache:' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://google.com" alt="go">Google cache</a>' . + '<a href="https://web.archive.org/web/' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.org" alt="ar">Archive.org</a>' . + '<a href="https://archive.is/newest/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.is" alt="ar">Archive.is</a>' . + '<a href="https://www.bing.com/search?q=url%3A' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://bing.com" alt="bi">Bing cache</a>' . + '<a href="https://megalodon.jp/?url=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://megalodon.jp" alt="me">Megalodon</a>' . + '</div>'; + + /* + Draw link + */ + $parts = explode("/", $link); + $clickurl = ""; + + // remove trailing / + $c = count($parts) - 1; + if($parts[$c] == ""){ + + $parts[$c - 1] = $parts[$c - 1] . "/"; + unset($parts[$c]); + } + + // merge https://site together + $parts = [ + $parts[0] . $parts[1] . '//' . $parts[2], + ...array_slice($parts, 3, count($parts) - 1) + ]; + + $c = count($parts); + for($i=0; $i<$c; $i++){ + + if($i !== 0){ $clickurl .= "/"; } + + $clickurl .= $parts[$i]; + + if($i === $c - 1){ + + $parts[$i] = rtrim($parts[$i], "/"); + } + + $payload .= + '<a class="part" href="' . htmlspecialchars($clickurl) . '" rel="noreferrer nofollow" tabindex="-1">' . + htmlspecialchars(urldecode($parts[$i])) . + '</a>'; + + if($i !== $c - 1){ + + $payload .= '<span class="separator"></span>'; + } + } + + return $payload . '</div>'; + } + + public function getscraperfilters($page){ + + $get_scraper = null; + + switch($page){ + + case "web": + $get_scraper = isset($_COOKIE["scraper_web"]) ? $_COOKIE["scraper_web"] : null; + break; + + case "images": + $get_scraper = isset($_COOKIE["scraper_images"]) ? $_COOKIE["scraper_images"] : null; + break; + + case "videos": + $get_scraper = isset($_COOKIE["scraper_videos"]) ? $_COOKIE["scraper_videos"] : null; + break; + + case "news": + $get_scraper = isset($_COOKIE["scraper_news"]) ? $_COOKIE["scraper_news"] : null; + break; + } + + if( + isset($_GET["scraper"]) && + is_string($_GET["scraper"]) + ){ + + $get_scraper = $_GET["scraper"]; + }else{ + + if( + isset($_GET["npt"]) && + is_string($_GET["npt"]) + ){ + + $get_scraper = explode(".", $_GET["npt"], 2)[0]; + + $get_scraper = + preg_replace( + '/[0-9]+$/', + "", + $get_scraper + ); + } + } + + // add search field + $filters = + [ + "s" => [ + "option" => "_SEARCH" + ] + ]; + + // define default scrapers + switch($page){ + + case "web": + $filters["scraper"] = [ + "display" => "Scraper", + "option" => [ + "ddg" => "DuckDuckGo", + "brave" => "Brave", + "google" => "Google", + "mojeek" => "Mojeek", + "marginalia" => "Marginalia", + "wiby" => "wiby" + ] + ]; + break; + + case "images": + $filters["scraper"] = [ + "display" => "Scraper", + "option" => [ + "ddg" => "DuckDuckGo", + "yandex" => "Yandex", + "google" => "Google" + ] + ]; + break; + + case "videos": + $filters["scraper"] = [ + "display" => "Scraper", + "option" => [ + "yt" => "YouTube", + "ddg" => "DuckDuckGo", + "google" => "Google" + ] + ]; + break; + + case "news": + $filters["scraper"] = [ + "display" => "Scraper", + "option" => [ + "ddg" => "DuckDuckGo", + "brave" => "Brave", + "google" => "Google", + "mojeek" => "Mojeek" + ] + ]; + break; + } + + // get scraper name from user input, or default out to preferred scraper + $scraper_out = null; + $first = true; + + foreach($filters["scraper"]["option"] as $scraper_name => $scraper_pretty){ + + if($first === true){ + + $first = $scraper_name; + } + + if($scraper_name == $get_scraper){ + + $scraper_out = $scraper_name; + } + } + + if($scraper_out === null){ + + $scraper_out = $first; + } + + switch($scraper_out){ + + case "ddg": + include "scraper/ddg.php"; + $lib = new ddg(); + break; + + case "brave": + include "scraper/brave.php"; + $lib = new brave(); + break; + + case "yt"; + include "scraper/youtube.php"; + $lib = new youtube(); + break; + + case "yandex": + include "scraper/yandex.php"; + $lib = new yandex(); + break; + + case "google": + include "scraper/google.php"; + $lib = new google(); + break; + + case "mojeek": + include "scraper/mojeek.php"; + $lib = new mojeek(); + break; + + case "marginalia": + include "scraper/marginalia.php"; + $lib = new marginalia(); + break; + + case "wiby": + include "scraper/wiby.php"; + $lib = new wiby(); + break; + } + + // set scraper on $_GET + $_GET["scraper"] = $scraper_out; + + // set nsfw on $_GET + if( + isset($_COOKIE["nsfw"]) && + !isset($_GET["nsfw"]) + ){ + + $_GET["nsfw"] = $_COOKIE["nsfw"]; + } + + return + [ + $lib, + array_merge_recursive( + $filters, + $lib->getfilters($page) + ) + ]; + } + + public function parsegetfilters($parameters, $whitelist){ + + $sanitized = []; + + // add npt token + if( + isset($parameters["npt"]) && + is_string($parameters["npt"]) + ){ + + $sanitized["npt"] = $parameters["npt"]; + }else{ + + $sanitized["npt"] = false; + } + + // we're iterating over $whitelist, so + // you can't polluate $sanitized with useless + // parameters + foreach($whitelist as $parameter => $value){ + + if(isset($parameters[$parameter])){ + + if(!is_string($parameters[$parameter])){ + + $sanitized[$parameter] = null; + continue; + } + + // parameter is already set, use that value + $sanitized[$parameter] = $parameters[$parameter]; + }else{ + + // parameter is not set, add it + if(is_string($value["option"])){ + + // special field: set default value manually + switch($value["option"]){ + + case "_DATE": + // no date set + $sanitized[$parameter] = false; + break; + + case "_SEARCH": + // no search set + $sanitized[$parameter] = ""; + break; + } + + }else{ + + // set a default value + $sanitized[$parameter] = array_keys($value["option"])[0]; + } + } + + // sanitize input + if(is_array($value["option"])){ + if( + !in_array( + $sanitized[$parameter], + $keys = array_keys($value["option"]) + ) + ){ + + $sanitized[$parameter] = $keys[0]; + } + }else{ + + // sanitize search & string + switch($value["option"]){ + + case "_DATE": + if($sanitized[$parameter] !== false){ + + $sanitized[$parameter] = strtotime($sanitized[$parameter]); + if($sanitized[$parameter] <= 0){ + + $sanitized[$parameter] = false; + } + } + break; + + case "_SEARCH": + + // get search string & bang + $sanitized[$parameter] = trim($sanitized[$parameter]); + $sanitized["bang"] = ""; + + if( + strlen($sanitized[$parameter]) !== 0 && + $sanitized[$parameter][0] == "!" + ){ + + $sanitized[$parameter] = explode(" ", $sanitized[$parameter], 2); + + $sanitized["bang"] = trim($sanitized[$parameter][0]); + + if(count($sanitized[$parameter]) === 2){ + + $sanitized[$parameter] = trim($sanitized[$parameter][1]); + }else{ + + $sanitized[$parameter] = ""; + } + + $sanitized["bang"] = ltrim($sanitized["bang"], "!"); + } + + $sanitized[$parameter] = ltrim($sanitized[$parameter], "! \n\r\t\v\x00"); + } + } + } + + // invert dates if needed + if( + isset($sanitized["older"]) && + isset($sanitized["newer"]) && + $sanitized["newer"] !== false && + $sanitized["older"] !== false && + $sanitized["newer"] > $sanitized["older"] + ){ + + // invert + [ + $sanitized["older"], + $sanitized["newer"] + ] = [ + $sanitized["newer"], + $sanitized["older"] + ]; + } + + return $sanitized; + } + + public function s_to_timestamp($seconds){ + + if(is_string($seconds)){ + + return "LIVE"; + } + + return ($seconds >= 60) ? ltrim(gmdate("H:i:s", $seconds), ":0") : gmdate("0:s", $seconds); + } + + public function generatehtmltabs($page, $query){ + + $html = null; + + foreach(["web", "images", "videos", "news"] as $type){ + + $html .= '<a href="/' . $type . '?s=' . urlencode($query); + + if(!empty($params)){ + + $html .= $params; + } + + $html .= '" class="tab'; + + if($type == $page){ + + $html .= ' selected'; + } + + $html .= '">' . ucfirst($type) . '</a>'; + } + + return $html; + } + + public function generatehtmlfilters($filters, $params){ + + $html = null; + + foreach($filters as $filter_name => $filter_values){ + + if(!isset($filter_values["display"])){ + + continue; + } + + $output = true; + $tmp = + '<div class="filter">' . + '<div class="title">' . htmlspecialchars($filter_values["display"]) . '</div>'; + + if(is_array($filter_values["option"])){ + + $tmp .= '<select name="' . $filter_name . '">'; + + foreach($filter_values["option"] as $option_name => $option_title){ + + $tmp .= '<option value="' . $option_name . '"'; + + if($params[$filter_name] == $option_name){ + + $tmp .= ' selected'; + } + + $tmp .= '>' . htmlspecialchars($option_title) . '</option>'; + } + + $tmp .= '</select>'; + }else{ + + switch($filter_values["option"]){ + + case "_DATE": + $tmp .= '<input type="date" name="' . $filter_name . '"'; + + if($params[$filter_name] !== false){ + + $tmp .= ' value="' . date("Y-m-d", $params[$filter_name]) . '"'; + } + + $tmp .= '>'; + break; + + default: + $output = false; + break; + } + } + + $tmp .= '</div>'; + + if($output === true){ + + $html .= $tmp; + } + } + + return $html; + } + + public function buildquery($gets, $ommit = false){ + + $out = []; + foreach($gets as $key => $value){ + + if( + $value == null || + $value == false || + $key == "npt" || + $key == "extendedsearch" || + $value == "any" || + $value == "all" || + ( + $ommit === true && + $key == "s" + ) + ){ + + continue; + } + + $out[$key] = $value; + } + + return http_build_query($out); + } + + public function htmlnextpage($gets, $npt, $page){ + + $query = $this->buildquery($gets); + + return $page . "?" . $query . "&npt=" . $npt; + } +} diff --git a/lib/fuckhtml.php b/lib/fuckhtml.php new file mode 100644 index 0000000..8802511 --- /dev/null +++ b/lib/fuckhtml.php @@ -0,0 +1,361 @@ +<?php +class fuckhtml{ + + public function __construct($html = null, $isfile = false){ + + if($html !== null){ + + $this->load($html, $isfile); + } + } + + public function load($html, $isfile = false){ + + if(is_array($html)){ + + if(!isset($html["innerHTML"])){ + + throw new Exception("(load) Supplied array doesn't contain a innerHTML index"); + } + $html = $html["innerHTML"]; + } + + if($isfile){ + + $handle = fopen($html, "r"); + $fetch = fread($handle, filesize($html)); + fclose($handle); + + $this->html = $fetch; + }else{ + + $this->html = $html; + } + + $this->strlen = strlen($this->html); + } + + public function getElementsByTagName(string $tagname){ + + $out = []; + + /* + Scrape start of the tag. Example + <div class="mydiv"> ... + */ + + if($tagname == "*"){ + + $tagname = '[^\/<>\s]+'; + }else{ + + $tagname = preg_quote(strtolower($tagname)); + } + + preg_match_all( + '/<\s*(' . $tagname . ')(\s(?:[^>\'"]*|"[^"]*"|\'[^\']*\')+)?\s*>/i', + /* '/<\s*(' . $tagname . ')(\s[\S\s]*?)?>/i', */ + $this->html, + $starting_tags, + PREG_OFFSET_CAPTURE + ); + + for($i=0; $i<count($starting_tags[0]); $i++){ + + /* + Parse attributes + */ + $attributes = []; + + preg_match_all( + '/([^\/\s\\=]+)(?:\s*=\s*("[^"]*"|\'[^\']*\'|[^\s]*))?/', + $starting_tags[2][$i][0], + $regex_attributes + ); + + for($k=0; $k<count($regex_attributes[0]); $k++){ + + if(trim($regex_attributes[2][$k]) == ""){ + + $attributes[$regex_attributes[1][$k]] = + "true"; + + continue; + } + + $attributes[$regex_attributes[1][$k]] = + trim($regex_attributes[2][$k], "'\" \n\r\t\v\x00"); + } + + $out[] = [ + "tagName" => strtolower($starting_tags[1][$i][0]), + "startPos" => $starting_tags[0][$i][1], + "endPos" => 0, + "startTag" => $starting_tags[0][$i][0], + "attributes" => $attributes, + "innerHTML" => null + ]; + } + + /* + Get innerHTML + */ + // get closing tag positions + preg_match_all( + '/<\s*\/\s*(' . $tagname . ')\s*>/i', + $this->html, + $regex_closing_tags, + PREG_OFFSET_CAPTURE + ); + + // merge opening and closing tags together + for($i=0; $i<count($regex_closing_tags[1]); $i++){ + + $out[] = [ + "tagName" => strtolower($regex_closing_tags[1][$i][0]), + "endTag" => $regex_closing_tags[0][$i][0], + "startPos" => $regex_closing_tags[0][$i][1] + ]; + } + + usort( + $out, + function($a, $b){ + + return $a["startPos"] > $b["startPos"]; + } + ); + + // computer the indent level for each element + $level = []; + $count = count($out); + + for($i=0; $i<$count; $i++){ + + if(!isset($level[$out[$i]["tagName"]])){ + + $level[$out[$i]["tagName"]] = 0; + } + + if(isset($out[$i]["startTag"])){ + + // encountered starting tag + $level[$out[$i]["tagName"]]++; + $out[$i]["level"] = $level[$out[$i]["tagName"]]; + }else{ + + // encountered closing tag + $out[$i]["level"] = $level[$out[$i]["tagName"]]; + $level[$out[$i]["tagName"]]--; + } + } + + // if the indent level is the same for a div, + // we encountered _THE_ closing tag + for($i=0; $i<$count; $i++){ + + if(!isset($out[$i]["startTag"])){ + + continue; + } + + for($k=$i; $k<$count; $k++){ + + if( + isset($out[$k]["endTag"]) && + $out[$i]["tagName"] == $out[$k]["tagName"] && + $out[$i]["level"] + === $out[$k]["level"] + ){ + + $startlen = strlen($out[$i]["startTag"]); + $endlen = strlen($out[$k]["endTag"]); + + $out[$i]["endPos"] = $out[$k]["startPos"] + $endlen; + + $out[$i]["innerHTML"] = + substr( + $this->html, + $out[$i]["startPos"] + $startlen, + $out[$k]["startPos"] - ($out[$i]["startPos"] + $startlen) + ); + + $out[$i]["outerHTML"] = + substr( + $this->html, + $out[$i]["startPos"], + $out[$k]["startPos"] - $out[$i]["startPos"] + $endlen + ); + + break; + } + } + } + + // filter out ending divs + for($i=0; $i<$count; $i++){ + + if(isset($out[$i]["endTag"])){ + + unset($out[$i]); + } + + unset($out[$i]["startTag"]); + } + + return array_values($out); + } + + public function getElementsByAttributeName(string $name, $collection = null){ + + if($collection === null){ + + $collection = $this->getElementsByTagName("*"); + }elseif(is_string($collection)){ + + $collection = $this->getElementsByTagName($collection); + } + + $return = []; + foreach($collection as $elem){ + + foreach($elem["attributes"] as $attrib_name => $attrib_value){ + + if($attrib_name == $name){ + + $return[] = $elem; + continue 2; + } + } + } + + return $return; + } + + public function getElementsByFuzzyAttributeValue(string $name, string $value, $collection = null){ + + $elems = $this->getElementsByAttributeName($name, $collection); + $value = explode(" ", $value); + + $return = []; + + foreach($elems as $elem){ + + foreach($elem["attributes"] as $attrib_name => $attrib_value){ + + $attrib_value = explode(" ", $attrib_value); + $ac = count($attrib_value); + $nc = count($value); + $cr = 0; + + for($i=0; $i<$nc; $i++){ + + for($k=0; $k<$ac; $k++){ + + if($value[$i] == $attrib_value[$k]){ + + $cr++; + } + } + } + + if($cr === $nc){ + + $return[] = $elem; + continue 2; + } + } + } + + return $return; + } + + public function getElementsByAttributeValue(string $name, string $value, $collection = null){ + + $elems = $this->getElementsByAttributeName($name, $collection); + + $return = []; + + foreach($elems as $elem){ + + foreach($elem["attributes"] as $attrib_name => $attrib_value){ + + if($attrib_value == $value){ + + $return[] = $elem; + continue 2; + } + } + } + + return $return; + } + + public function getElementById(string $idname, $collection = null){ + + $id = $this->getElementsByAttributeValue("id", $idname, $collection); + + if(count($id) !== 0){ + + return $id[0]; + } + + return false; + } + + public function getElementsByClassName(string $classname, $collection = null){ + + return $this->getElementsByFuzzyAttributeValue("class", $classname, $collection); + } + + public function getTextContent($html, $whitespace = false, $trim = true){ + + if(is_array($html)){ + + if(!isset($html["innerHTML"])){ + + throw new Exception("(getTextContent) Supplied array doesn't contain a innerHTML index"); + } + $html = $html["innerHTML"]; + } + + $html = + preg_split('/\n|<\/?br>/i', $html); + + $out = ""; + for($i=0; $i<count($html); $i++){ + + $tmp = + html_entity_decode( + strip_tags( + $html[$i] + ), + ENT_QUOTES | ENT_XML1, "UTF-8" + ); + + if($trim){ + + $tmp = trim($tmp); + } + + $out .= $tmp; + + if($whitespace === true){ + + $out .= "\n"; + }else{ + + $out .= " "; + } + } + + if($trim){ + + return trim($out); + } + + return $out; + } +} + +?> diff --git a/lib/img404.png b/lib/img404.png Binary files differnew file mode 100644 index 0000000..4549dee --- /dev/null +++ b/lib/img404.png diff --git a/lib/nextpage.php b/lib/nextpage.php new file mode 100644 index 0000000..a883e49 --- /dev/null +++ b/lib/nextpage.php @@ -0,0 +1,106 @@ +<?php + +class nextpage{ + + public function __construct($scraper){ + + $this->scraper = $scraper; + } + + public function store($payload, $page){ + + $page = $page[0]; + $password = random_bytes(256); // 2048 bit + $salt = random_bytes(16); + $key = hash_pbkdf2("sha512", $password, $salt, 20000, 32, true); + $iv = + random_bytes( + openssl_cipher_iv_length("aes-256-gcm") + ); + + $tag = ""; + $out = openssl_encrypt($payload, "aes-256-gcm", $key, OPENSSL_RAW_DATA, $iv, $tag, "", 16); + + $key = apcu_inc("key", 1); + + apcu_store( + $page . "." . + $this->scraper . + (string)($key), + gzdeflate($salt.$iv.$out.$tag), + 420 // cache information for 7 minutes blaze it + ); + + return + $this->scraper . $key . "." . + rtrim(strtr(base64_encode($password), '+/', '-_'), '='); + } + + public function get($npt, $page){ + + $page = $page[0]; + $explode = explode(".", $npt, 2); + + if(count($explode) !== 2){ + + throw new Exception("Malformed nextPageToken!"); + } + + $apcu = $page . "." . $explode[0]; + $key = $explode[1]; + + $payload = apcu_fetch($apcu); + + if($payload === false){ + + throw new Exception("The nextPageToken is invalid or has expired!"); + } + + $key = + base64_decode( + str_pad( + strtr($key, '-_', '+/'), + strlen($key) % 4, + '=', + STR_PAD_RIGHT + ) + ); + + $payload = gzinflate($payload); + + $key = + hash_pbkdf2( + "sha512", + $key, + substr($payload, 0, 16), // salt + 20000, + 32, + true + ); + $ivlen = openssl_cipher_iv_length("aes-256-gcm"); + + $payload = + openssl_decrypt( + substr( + $payload, + 16 + $ivlen, + -16 + ), + "aes-256-gcm", + $key, + OPENSSL_RAW_DATA, + substr($payload, 16, $ivlen), + substr($payload, -16) + ); + + if($payload === false){ + + throw new Exception("The nextPageToken is invalid or has expired!"); + } + + // remove the key after using + apcu_delete($apcu); + + return $payload; + } +} diff --git a/lib/type-todo.php b/lib/type-todo.php new file mode 100644 index 0000000..f813543 --- /dev/null +++ b/lib/type-todo.php @@ -0,0 +1,132 @@ + + public function type($get){ + + $search = $get["s"]; + $bang = $get["bang"]; + + if(empty($search)){ + + if(!empty($bang)){ + + // !youtube + $conn = pg_connect("host=localhost dbname=4get user=postgres password=postgres"); + + pg_prepare($conn, "bang_get", "SELECT bang,name FROM bangs WHERE bang LIKE $1 ORDER BY bang ASC LIMIT 8"); + $q = pg_execute($conn, "bang_get", ["$bang%"]); + + $results = []; + while($row = pg_fetch_array($q, null, PGSQL_ASSOC)){ + + $results[] = [ + "s" => "!" . $row["bang"], + "n" => $row["name"] + ]; + } + + return $results; + }else{ + + // everything is empty + // lets just return a bang list + return [ + [ + "s" => "!w", + "n" => "Wikipedia", + "u" => "https://en.wikipedia.org/wiki/Special:Search?search={%q%}" + ], + [ + "s" => "!4ch", + "n" => "4chan Board", + "u" => "https://find.4chan.org/?q={%q%}" + ], + [ + "s" => "!a", + "n" => "Amazon", + "u" => "https://www.amazon.com/s?k={%q%}" + ], + [ + "s" => "!e", + "n" => "eBay", + "u" => "https://www.ebay.com/sch/items/?_nkw={%q%}" + ], + [ + "s" => "!so", + "n" => "Stack Overflow", + "u" => "http://stackoverflow.com/search?q={%q%}" + ], + [ + "s" => "!gh", + "n" => "GitHub", + "u" => "https://github.com/search?utf8=%E2%9C%93&q={%q%}" + ], + [ + "s" => "!tw", + "n" => "Twitter", + "u" => "https://twitter.com/search?q={%q%}" + ], + [ + "s" => "!r", + "n" => "Reddit", + "u" => "https://www.reddit.com/search?q={%q%}" + ], + ]; + } + } + + // now we know search isnt empty + if(!empty($bang)){ + + // check if the bang exists + $conn = pg_connect("host=localhost dbname=4get user=postgres password=postgres"); + + pg_prepare($conn, "bang_get_single", "SELECT bang,name FROM bangs WHERE bang = $1 LIMIT 1"); + $q = pg_execute($conn, "bang_get_single", [$bang]); + + $row = pg_fetch_array($q, null, PGSQL_ASSOC); + + if(isset($row["bang"])){ + + $bang = "!$bang "; + }else{ + + $bang = ""; + } + } + + try{ + $res = $this->get( + "https://duckduckgo.com/ac/", + [ + "q" => strtolower($search) + ], + ddg::req_xhr + ); + + $res = json_decode($res, true); + + }catch(Exception $e){ + + throw new Exception("Failed to get /ac/"); + } + + $arr = []; + for($i=0; $i<count($res); $i++){ + + if($i === 8){break;} + + if(empty($bang)){ + + $arr[] = [ + "s" => $res[$i]["phrase"] + ]; + }else{ + + $arr[] = [ + "s" => $bang . $res[$i]["phrase"], + "n" => $row["name"] + ]; + } + } + + return $arr; + } diff --git a/news.php b/news.php new file mode 100644 index 0000000..eb5817f --- /dev/null +++ b/news.php @@ -0,0 +1,96 @@ +<?php + +/* + Initialize random shit +*/ +include "lib/frontend.php"; +$frontend = new frontend(); + +[$scraper, $filters] = $frontend->getscraperfilters("news"); + +$get = $frontend->parsegetfilters($_GET, $filters); + +$frontend->loadheader( + $get, + $filters, + "news" +); + +$payload = [ + "class" => "", + "right-left" => "", + "right-right" => "", + "left" => "" +]; + +try{ + $results = $scraper->news($get); + +}catch(Exception $error){ + + echo + $frontend->drawerror( + "Shit", + 'This scraper returned an error:' . + '<div class="code">' . htmlspecialchars($error->getMessage()) . '</div>' . + 'Things you can try:' . + '<ul>' . + '<li>Use a different scraper</li>' . + '<li>Remove keywords that could cause errors</li>' . + '<li>Use another 4get instance</li>' . + '</ul><br>' . + 'If the error persists, please <a href="/about">contact the administrator</a>.' + ); + die(); +} + +/* + Populate links +*/ +if(count($results["news"]) === 0){ + + $payload["left"] = + '<div class="infobox">' . + "<h1>Nobody here but us chickens!</h1>" . + 'Have you tried:' . + '<ul>' . + '<li>Using a different scraper</li>' . + '<li>Using fewer keywords</li>' . + '<li>Defining broader filters (Is NSFW turned off?)</li>' . + '</ul>' . + '</div>'; +} + +foreach($results["news"] as $news){ + + $greentext = []; + + if($news["date"] !== null){ + + $greentext[] = date("jS M y @ g:ia", $news["date"]); + } + + if($news["author"] !== null){ + + $greentext[] = htmlspecialchars($news["author"]); + } + + if(count($greentext) !== 0){ + + $greentext = implode(" • ", $greentext); + }else{ + + $greentext = null; + } + + $n = null; + $payload["left"] .= $frontend->drawtextresult($news, $greentext, $n, $get["s"]); +} + +if($results["npt"] !== null){ + + $payload["left"] .= + '<a href="' . $frontend->htmlnextpage($get, $results["npt"], "news") . '" class="nextpage">Next page ></a>'; +} + +echo $frontend->load("search.html", $payload); diff --git a/opensearch.xml b/opensearch.xml new file mode 100644 index 0000000..efce4b4 --- /dev/null +++ b/opensearch.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/"> +<ShortName>4get</ShortName> + +<InputEncoding>UTF-8</InputEncoding> +<Image height="16" width="16">https://4get.ca/favicon.ico</Image> +<Url type="text/html" method="GET" template="https://4get.ca/web?s={searchTerms}"/> + +</OpenSearchDescription> diff --git a/proxy.php b/proxy.php new file mode 100644 index 0000000..edefd77 --- /dev/null +++ b/proxy.php @@ -0,0 +1,130 @@ +<?php + +include "lib/curlproxy.php"; +$proxy = new proxy(); + +if(!isset($_GET["i"])){ + + header("X-Error: No URL(i) provided!"); + $proxy->do404(); + die(); +} + +try{ + + // original size request, stream file to browser + if( + !isset($_GET["s"]) || + $_GET["s"] == "original" + ){ + + $proxy->stream_linear_image($_GET["i"]); + die(); + } + + // bing request, ask bing to resize and stream to browser + if( + preg_match( + '/bing.net$/', + parse_url($_GET["i"], PHP_URL_HOST) + ) + ){ + + switch($_GET["s"]){ + + case "portrait": $req = "&w=50&h=90&p=0&qlt=99"; break; + case "landscape": $req = "&w=160&h=90&p=0&qlt=99"; break; + case "square": $req = "&w=90&h=90&p=0&qlt=99"; break; + case "thumb": $req = "&w=236&h=180&p=0&qlt=99"; break; + case "cover": $req = "&w=207&h=270&p=0&qlt=99"; break; + } + + $proxy->stream_linear_image($_GET["i"] . $req, "https://bing.net"); + die(); + } + + // resize image ourselves + $payload = $proxy->get($_GET["i"], $proxy::req_image, true); + + // get image format & set imagick + $image = null; + $format = $proxy->getimageformat($payload, $image); + + try{ + + if($format !== false){ + $image->setFormat($format); + } + + $image->readImageBlob($payload["body"]); + $image_width = $image->getImageWidth(); + $image_height = $image->getImageHeight(); + + switch($_GET["s"]){ + + case "portrait": + $width = 50; + $height = 90; + break; + + case "landscape": + $width = 160; + $height = 90; + break; + + case "square": + $width = 90; + $height = 90; + break; + + case "thumb": + $width = 236; + $height = 180; + break; + + case "cover": + $width = 207; + $height = 270; + break; + } + + $ratio = $image_width / $image_height; + + if($image_width > $width){ + + $image_width = $width; + $image_height = round($image_width / $ratio); + } + if($image_height > $height){ + + $ratio = $image_width / $image_height; + $image_height = $height; + $image_width = $image_height * $ratio; + } + + $image->setImageBackgroundColor(new ImagickPixel("#504945")); + $image->mergeImageLayers(Imagick::LAYERMETHOD_FLATTEN); + + $image->resizeImage($image_width, $image_height, Imagick::FILTER_LANCZOS, 1); + + $image->stripImage(); + $image->setFormat("jpeg"); + $image->setImageCompression(Imagick::COMPRESSION_JPEG2000); + + $proxy->getfilenameheader($payload["headers"], $_GET["i"]); + + header("Content-Type: image/jpeg"); + echo $image->getImageBlob(); + + }catch(ImagickException $error){ + + header("X-Error: Could not convert the image: (" . $error->getMessage() . ")"); + $proxy->do404(); + } + +}catch(Exception $error){ + + header("X-Error: " . $error->getMessage()); + $proxy->do404(); + die(); +} diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..3e608cc --- /dev/null +++ b/robots.txt @@ -0,0 +1,28 @@ +# When the robots.txt is sus + +# ⠀⠀⠀⡯⡯⡾⠝⠘⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢊⠘⡮⣣⠪⠢⡑⡌ +# ⠀⠀⠀⠟⠝⠈⠀⠀⠀⠡⠀⠠⢈⠠⢐⢠⢂⢔⣐⢄⡂⢔⠀⡁⢉⠸⢨⢑⠕⡌ +# ⠀⠀⡀⠁⠀⠀⠀⡀⢂⠡⠈⡔⣕⢮⣳⢯⣿⣻⣟⣯⣯⢷⣫⣆⡂⠀⠀⢐⠑⡌ +# ⢀⠠⠐⠈⠀⢀⢂⠢⡂⠕⡁⣝⢮⣳⢽⡽⣾⣻⣿⣯⡯⣟⣞⢾⢜⢆⠀⡀⠀⠪ +# ⣬⠂⠀⠀⢀⢂⢪⠨⢂⠥⣺⡪⣗⢗⣽⢽⡯⣿⣽⣷⢿⡽⡾⡽⣝⢎⠀⠀⠀⢡ +# ⣿⠀⠀⠀⢂⠢⢂⢥⢱⡹⣪⢞⡵⣻⡪⡯⡯⣟⡾⣿⣻⡽⣯⡻⣪⠧⠑⠀⠁⢐ +# ⣿⠀⠀⠀⠢⢑⠠⠑⠕⡝⡎⡗⡝⡎⣞⢽⡹⣕⢯⢻⠹⡹⢚⠝⡷⡽⡨⠀⠀⢔ +# ⣿⡯⠀⢈⠈⢄⠂⠂⠐⠀⠌⠠⢑⠱⡱⡱⡑⢔⠁⠀⡀⠐⠐⠐⡡⡹⣪⠀⠀⢘ +# ⣿⣽⠀⡀⡊⠀⠐⠨⠈⡁⠂⢈⠠⡱⡽⣷⡑⠁⠠⠑⠀⢉⢇⣤⢘⣪⢽⠀⢌⢎ +# ⣿⢾⠀⢌⠌⠀⡁⠢⠂⠐⡀⠀⢀⢳⢽⣽⡺⣨⢄⣑⢉⢃⢭⡲⣕⡭⣹⠠⢐⢗ +# ⣿⡗⠀⠢⠡⡱⡸⣔⢵⢱⢸⠈⠀⡪⣳⣳⢹⢜⡵⣱⢱⡱⣳⡹⣵⣻⢔⢅⢬⡷ +# ⣷⡇⡂⠡⡑⢕⢕⠕⡑⠡⢂⢊⢐⢕⡝⡮⡧⡳⣝⢴⡐⣁⠃⡫⡒⣕⢏⡮⣷⡟ +# ⣷⣻⣅⠑⢌⠢⠁⢐⠠⠑⡐⠐⠌⡪⠮⡫⠪⡪⡪⣺⢸⠰⠡⠠⠐⢱⠨⡪⡪⡰ +# ⣯⢷⣟⣇⡂⡂⡌⡀⠀⠁⡂⠅⠂⠀⡑⡄⢇⠇⢝⡨⡠⡁⢐⠠⢀⢪⡐⡜⡪⡊ +# ⣿⢽⡾⢹⡄⠕⡅⢇⠂⠑⣴⡬⣬⣬⣆⢮⣦⣷⣵⣷⡗⢃⢮⠱⡸⢰⢱⢸⢨⢌ +# ⣯⢯⣟⠸⣳⡅⠜⠔⡌⡐⠈⠻⠟⣿⢿⣿⣿⠿⡻⣃⠢⣱⡳⡱⡩⢢⠣⡃⠢⠁ +# ⡯⣟⣞⡇⡿⣽⡪⡘⡰⠨⢐⢀⠢⢢⢄⢤⣰⠼⡾⢕⢕⡵⣝⠎⢌⢪⠪⡘⡌⠀ +# ⡯⣳⠯⠚⢊⠡⡂⢂⠨⠊⠔⡑⠬⡸⣘⢬⢪⣪⡺⡼⣕⢯⢞⢕⢝⠎⢻⢼⣀⠀ +# ⠁⡂⠔⡁⡢⠣⢀⠢⠀⠅⠱⡐⡱⡘⡔⡕⡕⣲⡹⣎⡮⡏⡑⢜⢼⡱⢩⣗⣯⣟ +# ⢀⢂⢑⠀⡂⡃⠅⠊⢄⢑⠠⠑⢕⢕⢝⢮⢺⢕⢟⢮⢊⢢⢱⢄⠃⣇⣞⢞⣞⢾ +# ⢀⠢⡑⡀⢂⢊⠠⠁⡂⡐⠀⠅⡈⠪⠪⠪⠣⠫⠑⡁⢔⠕⣜⣜⢦⡰⡎⡯⡾⡽ + +User-agent: * +Disallow: +host: 4get.ca +sitemap: https://4get.ca/sitemap.xml diff --git a/scraper/brave.php b/scraper/brave.php new file mode 100644 index 0000000..4d48c33 --- /dev/null +++ b/scraper/brave.php @@ -0,0 +1,2287 @@ +<?php +/* +$brave = new brave(); + +$handle = fopen("captcha.html", "r"); +$html = fread($handle, filesize("captcha.html")); +fclose($handle); + +$brave->bypasscaptcha($html, "yes", "ca");*/ + +class brave{ + + public function __construct(){ + + include "lib/fuckhtml.php"; + $this->fuckhtml = new fuckhtml(); + + include "lib/nextpage.php"; + $this->nextpage = new nextpage("brave"); + } + + public function getfilters($page){ + + switch($page){ + + case "web": + return [ + "country" => [ + "display" => "Country", + "option" => [ + "all" => "All Regions", + "ar" => "Argentina", + "au" => "Australia", + "at" => "Austria", + "be" => "Belgium", + "br" => "Brazil", + "ca" => "Canada", + "cl" => "Chile", + "cn" => "China", + "dk" => "Denmark", + "fi" => "Finland", + "fr" => "France", + "de" => "Germany", + "hk" => "Hong Kong", + "in" => "India", + "id" => "Indonesia", + "it" => "Italy", + "jp" => "Japan", + "kr" => "Korea", + "my" => "Malaysia", + "mx" => "Mexico", + "nl" => "Netherlands", + "nz" => "New Zealand", + "no" => "Norway", + "pl" => "Poland", + "pt" => "Portugal", + "ph" => "Philippines", + "ru" => "Russia", + "sa" => "Saudi Arabia", + "za" => "South Africa", + "es" => "Spain", + "se" => "Sweden", + "ch" => "Switzerland", + "tw" => "Taiwan", + "tr" => "Turkey", + "gb" => "United Kingdom", + "us" => "United States" + ] + ], + "nsfw" => [ + "display" => "NSFW", + "option" => [ + "yes" => "Yes", + "maybe" => "Maybe", + "no" => "No" + ] + ], + "newer" => [ + "display" => "Newer than", + "option" => "_DATE" + ], + "older" => [ + "display" => "Older than", + "option" => "_DATE" + ] + ]; + break; + + case "news": + return [ + "country" => [ + "display" => "Country", + "option" => [ + "all" => "All regions", + "ar" => "Argentina", + "au" => "Australia", + "at" => "Austria", + "be" => "Belgium", + "br" => "Brazil", + "ca" => "Canada", + "cl" => "Chile", + "cn" => "China", + "dk" => "Denmark", + "fi" => "Finland", + "fr" => "France", + "de" => "Germany", + "hk" => "Hong Kong", + "in" => "India", + "id" => "Indonesia", + "it" => "Italy", + "jp" => "Japan", + "kr" => "Korea", + "my" => "Malaysia", + "mx" => "Mexico", + "nl" => "Netherlands", + "nz" => "New Zealand", + "no" => "Norway", + "pl" => "Poland", + "pt" => "Portugal", + "ph" => "Philippines", + "ru" => "Russia", + "sa" => "Saudi Arabia", + "za" => "South Africa", + "es" => "Spain", + "se" => "Sweden", + "ch" => "Switzerland", + "tw" => "Taiwan", + "tr" => "Turkey", + "gb" => "United Kingdom", + "us" => "United States" + ] + ], + "nsfw" => [ + "display" => "NSFW", + "option" => [ + "yes" => "Yes", + "maybe" => "Maybe", + "no" => "No" + ] + ] + ]; + break; + } + } + + private function get($url, $get = [], $nsfw, $country/*, $is_post = false, $additional_cookies = null*/){ + + switch($nsfw){ + + case "yes": $nsfw = "off"; break; + case "maybe": $nsfw = "moderate"; break; + case "no": $nsfw = "strict"; break; + } + + //$cookie = "safesearch={$nsfw}; country={$country}; useLocation=0"; + /* + if($additional_cookies !== null){ + + $cookie = $additional_cookies . "; " . $cookie; + }*/ + + $headers = [ + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip", + "Cookie: safesearch={$nsfw}; country={$country}; useLocation=0; summarizer=0", + "DNT: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1"//, + //"Content-Type: application/json" + ]; + + if($country == "any"){ + + $country = "all"; + } + + $curlproc = curl_init(); + + /*if($is_post){ + + curl_setopt($curlproc, CURLOPT_POST, true); + curl_setopt( + $curlproc, + CURLOPT_POSTFIELDS, + json_encode($get) + ); + + }else{ + */ + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + //} + + curl_setopt($curlproc, CURLOPT_URL, $url); + + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + public function web($get){ + + if($get["npt"]){ + + // get next page data + $q = json_decode($this->nextpage->get($get["npt"], "web"), true); + + $search = $q["q"]; + $q["spellcheck"] = 0; + + $nsfw = $q["nsfw"]; + unset($q["nsfw"]); + + $country = $q["country"]; + unset($q["country"]); + + }else{ + + // get _GET data instead + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + if(strlen($search) > 2048){ + + throw new Exception("Search query is too long!"); + } + + $nsfw = $get["nsfw"]; + $country = $get["country"]; + $older = $get["older"]; + $newer = $get["newer"]; + + $q = [ + "q" => $search + ]; + + /* + Pass older/newer filters to brave + */ + if($newer !== false){ + + $newer = date("Y-m-d", $newer); + + if($older === false){ + + $older = date("Y-m-d", time()); + } + } + + if( + is_string($older) === false && + $older !== false + ){ + + $older = date("Y-m-d", $older); + + if($newer === false){ + + $newer = "1970-01-02"; + } + } + + if($older !== false){ + + $q["tf"] = "{$newer}to{$older}"; + } + } + /* + $handle = fopen("scraper/brave.html", "r"); + $html = fread($handle, filesize("scraper/brave.html")); + fclose($handle); + */ + try{ + $html = + $this->get( + "https://search.brave.com/search", + $q, + $nsfw, + $country + ); + + }catch(Exception $error){ + + throw new Exception("Could not fetch search page"); + } + + $out = [ + "status" => "ok", + "spelling" => [ + "type" => "no_correction", + "using" => null, + "correction" => null + ], + "npt" => null, + "answer" => [], + "web" => [], + "image" => [], + "video" => [], + "news" => [], + "related" => [] + ]; + + // load html + $this->fuckhtml->load($html); + + /* + Get next page "token" + */ + $nextpage = + $this->fuckhtml + ->getElementsByClassName( + "btn ml-15", + "a" + ); + + if(count($nextpage) !== 0){ + + preg_match( + '/offset=([0-9]+)/', + $this->fuckhtml->getTextContent($nextpage[0]["attributes"]["href"]), + $nextpage + ); + + $q["offset"] = (int)$nextpage[1]; + $q["nsfw"] = $nsfw; + $q["country"] = $country; + + $out["npt"] = + $this->nextpage->store( + json_encode($q), + "web" + ); + } + + /* + Get discussions (and append them to web results) + */ + + // they're loaded using javascript!! + $discussion = + $this->fuckhtml + ->getElementById( + "js-discussions", + "script" + ); + + if( + $discussion && + isset($discussion["attributes"]["data"]) + ){ + + $discussion = + json_decode( + $this->fuckhtml + ->getTextContent( + $discussion["attributes"]["data"] + ), + true + ); + + foreach($discussion["results"] as $result){ + + $data = [ + "title" => $this->titledots($result["title"]), + "description" => null, + "url" => $result["url"], + "date" => null, + "type" => "web", + "thumb" => [ + "url" => null, + "ratio" => null + ], + "sublink" => [], + "table" => [] + ]; + + // description + $data["description"] = + $this->limitstrlen( + $this->limitwhitespace( + $this->titledots( + $this->fuckhtml->getTextContent( + $result["description"] + ) + ) + ) + ); + + if($result["age"] != ""){ + $data["date"] = strtotime($result["age"]); + } + + // populate table + + if($result["data"]["num_answers"] != ""){ + $data["table"]["Replies"] = (int)$result["data"]["num_answers"]; + } + + if($result["data"]["score"] != ""){ + + $score = explode("|", $result["data"]["score"]); + + if(count($score) === 2){ + + $score = ((int)$score[1]) . " (" . trim($score[0]) . ")"; + }else{ + + $score = (int)$score[0]; + } + + $data["table"]["Votes"] = $score; + } + + if($result["thumbnail"] != ""){ + + $data["thumb"]["url"] = $result["thumbnail"]; + $data["thumb"]["ratio"] = "16:9"; + } + + $out["web"][] = $data; + } + } + + /* + Get related searches + */ + $faq = + $this->fuckhtml + ->getElementById("js-faq", "script"); + + if( + $faq && + isset($faq["attributes"]["data"]) + ){ + + $faq = + json_decode( + $this->fuckhtml + ->getTextContent( + $faq["attributes"]["data"] + ), + true + ); + + foreach($faq["items"] as $related){ + + $out["related"][] = $related["question"]; + } + } + + /* + Get spelling autocorrect + */ + $altered = + $this->fuckhtml + ->getElementById("altered-query", "div"); + + if($altered){ + + $this->fuckhtml->load($altered); + + $altered = + $this->fuckhtml + ->getElementsByTagName("a"); + + if(count($altered) === 2){ + + $out["spelling"] = [ + "type" => "including", + "using" => + $this->fuckhtml + ->getTextContent($altered[0]), + "correction" => + $this->fuckhtml + ->getTextContent($altered[1]) + ]; + } + + $this->fuckhtml->load($html); + } + + /* + Get web results + */ + $resulthtml = + $this->fuckhtml + ->getElementById( + "results", + "div" + ); + + $this->fuckhtml->load($resulthtml); + $items = 0; + foreach( + $this->fuckhtml + ->getElementsByClassName("snippet fdb") + as $result + ){ + + $data = [ + "title" => null, + "description" => null, + "url" => null, + "date" => null, + "type" => "web", + "thumb" => [ + "url" => null, + "ratio" => null + ], + "sublink" => [], + "table" => [] + ]; + + if( + isset($result["attributes"]["data-type"]) && + $result["attributes"]["data-type"] == "ad" + ){ + + // is an ad, skip + continue; + } + + $this->fuckhtml->load($result); + + /* + Get title + */ + $title = + $this->fuckhtml + ->getElementsByClassName( + "snippet-title", + "span" + ); + + if(count($title) === 0){ + + // encountered AI summarizer + // or misspelling indicator @TODO + continue; + } + + if(isset($title[0]["attributes"]["title"])){ + + $data["title"] = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $title[0]["attributes"]["title"] + ) + ); + }else{ + + $data["title"] = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $title[0] + ) + ); + } + + /* + Get description + */ + $description = + $this->fuckhtml + ->getElementsByClassName( + "snippet-description", + "p" + ); + + if(count($description) !== 0){ + $data["description"] = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $description[0] + ) + ); + + // also check for thumbnail in here + $img = + $this->fuckhtml + ->getElementsByClassName( + "thumb", + "img" + ); + + if(count($img) !== 0){ + + $data["thumb"] = [ + "url" => $this->unshiturl($img[0]["attributes"]["src"]), + "ratio" => "16:9" + ]; + }else{ + + // might be a video thumbnail wrapper? + $wrapper = + $this->fuckhtml + ->getElementsByClassName( + "video-thumb", + "a" + ); + + if(count($wrapper) !== 0){ + + // we found a video + $this->fuckhtml->load($wrapper[0]); + + $img = + $this->fuckhtml + ->getElementsByTagName("img"); + + $data["thumb"] = [ + "url" => $this->unshiturl($img[0]["attributes"]["src"]), + "ratio" => "16:9" + ]; + + // get the video length, if its there + $duration = + $this->fuckhtml + ->getElementsByClassName( + "duration", + "div" + ); + + if(count($duration) !== 0){ + + $data["table"]["Duration"] = $duration[0]["innerHTML"]; + } + + // reset html load + $this->fuckhtml->load($result); + } + } + + }else{ + + // is a steam/shop listing + $description_alt = + $this->fuckhtml + ->getElementsByClassName( + "text-sm", + "div" + ); + + if(count($description_alt) !== 0){ + + switch($description_alt[0]["attributes"]["class"]){ + + case "text-sm text-gray": + case "description text-sm": + + $data["description"] = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $description_alt[0] + ) + ); + break; + } + + // get table sublink + $sublink = + $this->fuckhtml + ->getElementsByClassName( + "r-attr text-sm", + "div" + ); + + if(count($sublink) !== 0){ + + $this->tablesublink($sublink, $data); + } + + // check for thumb element + $data["thumb"] = $this->getimagelinkfromstyle("thumb"); + }else{ + + // ok... finally... + // maybe its the instant answer thingy + $answer = + $this->fuckhtml + ->getElementsByClassName("answer"); + + if(count($answer) !== 0){ + + $data["description"] = + $this->titledots( + $this->fuckhtml + ->getTextContent($answer[0]) + ); + } + } + } + + // finally, fix brave's date format sucking balls + $data["description"] = explode(" - ", $data["description"], 2); + + if(count($data["description"]) === 0){ + + // nothing to do + $data["description"] = $data["description"][0]; + }else{ + + // attempt to parse + $time = strtotime($data["description"][0]); + + if($time !== false){ + + // got response + $data["date"] = $time; + + array_shift($data["description"]); + } + + // merge back + $data["description"] = + implode(" - ", $data["description"]); + } + + /* + Check content type + */ + $content_type = + $this->fuckhtml + ->getElementsByClassName( + "content-type", + "span" + ); + + if(count($content_type) !== 0){ + + $data["type"] = + strtolower($this->fuckhtml->getTextContent($content_type[0])); + } + + /* + Check subtext table thingy + */ + $table_items = + array_merge( + $this->fuckhtml + ->getElementsByClassName( + "item-attributes", + "div" + ), + $this->fuckhtml + ->getElementsByClassName( + "r", + "div" + ) + ); + + /* + DIV: item-attributes + */ + if(count($table_items) !== 0){ + + foreach($table_items as $table){ + + $this->fuckhtml->load($table); + + $span = + $this->fuckhtml + ->getElementsByClassName( + "text-sm", + "*" + ); + + foreach($span as $item){ + + $item = + explode( + ":", + $this->fuckhtml->getTextContent(preg_replace('/\n/', " ", $item["innerHTML"])), + 2 + ); + + if(count($item) === 2){ + + $data["table"][trim($item[0])] = trim($this->limitwhitespace($item[1])); + } + } + } + + $this->fuckhtml->load($result); + } + + // get video sublinks + $table_items = + $this->fuckhtml + ->getElementsByClassName( + "snippet-description published-time", + "p" + ); + + if(count($table_items) !== 0){ + + $table_items = + explode( + '<span class="mr-15"></span>', + $table_items[0]["innerHTML"], + 2 + ); + if(count($table_items) === 2){ + + $item2 = []; + + $item2[] = explode(":", $this->fuckhtml->getTextContent($table_items[0])); + + if(trim($table_items[1]) != ""){ + $item2[] = explode(":", $this->fuckhtml->getTextContent($table_items[1])); + } + + foreach($item2 as $it){ + + $data["table"][trim($it[0])] = trim($it[1]); + } + } + } + + /* + Get URL + */ + $data["url"] = + $this->fuckhtml->getTextContent( + $this->fuckhtml + ->getElementsByTagName("a") + [0] + ["attributes"] + ["href"] + ); + + /* + Get sublinks + */ + $sublinks_elems = + $this->fuckhtml + ->getElementsByClassName( + "snippet", + "div" + ); + + $sublinks = []; + + foreach($sublinks_elems as $sublink){ + + $this->fuckhtml->load($sublink); + + $a = + $this->fuckhtml + ->getElementsByTagName("a")[0]; + + $title = + $this->fuckhtml + ->getTextContent($a); + + $url = $a["attributes"]["href"]; + + $description = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByTagName("p")[0] + ) + ); + + $sublinks[] = [ + "title" => $title, + "date" => null, + "description" => $description, + "url" => $url + ]; + } + + /* + Get smaller sublinks + */ + $sublinks_elems = + $this->fuckhtml + ->getElementsByClassName( + "deep-link", + "a" + ); + + foreach($sublinks_elems as $sublink){ + + $sublinks[] = [ + "title" => $this->fuckhtml->getTextContent($sublink), + "date" => null, + "description" => null, + "url" => $sublink["attributes"]["href"] + ]; + } + + // append sublinks to $data !! + $data["sublink"] = $sublinks; + + // append first result to start of $out["web"] + // other results are after + if($items === 0){ + + $out["web"] = [$data, ...$out["web"]]; + }else{ + + $out["web"][] = $data; + } + $items++; + } + + /* + Get news + */ + $this->fuckhtml->load($resulthtml); + $news_carousel = $this->fuckhtml->getElementById("news-carousel"); + + $this->fuckhtml->load($news_carousel); + + if($news_carousel){ + + $a = + $this->fuckhtml + ->getElementsByClassName( + "card fdb", + "a" + ); + + foreach($a as $news){ + + $this->fuckhtml->load($news); + + $out["news"][] = [ + "title" => + $this->titledots( + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "title", + "div" + )[0] + ) + ), + "description" => null, + "date" => + strtotime( + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "card-footer__timestamp", + "span" + )[0] + ) + ), + "thumb" => $this->getimagelinkfromstyle("img-bg"), + "url" => $this->fuckhtml->getTextContent($news["attributes"]["href"]) + ]; + } + } + + + + /* + Get videos + */ + $this->fuckhtml->load($resulthtml); + $news_carousel = $this->fuckhtml->getElementById("video-carousel"); + + $this->fuckhtml->load($news_carousel); + + if($news_carousel){ + + $a = + $this->fuckhtml + ->getElementsByClassName( + "card fdb", + "a" + ); + + foreach($a as $video){ + + $this->fuckhtml->load($video); + + $date = null; + + $date_o = + $this->fuckhtml + ->getElementsByClassName( + "text-gray text-xs", + "span" + ); + + if(count($date_o) !== 0){ + + $date = + strtotime( + $this->fuckhtml + ->getTextContent( + $date_o[0] + ) + ); + } + + $out["video"][] = [ + "title" => + $this->titledots( + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "title", + "div" + )[0] + ) + ), + "description" => null, + "date" => $date, + "duration" => null, + "views" => null, + "thumb" => $this->getimagelinkfromstyle("img-bg"), + "url" => $this->fuckhtml->getTextContent($video["attributes"]["href"]) + ]; + } + } + + + /* + Get DEFINITION snippet + */ + $this->fuckhtml->load($html); + $infobox = $this->fuckhtml->getElementById("rh-definitions", "div"); + + if($infobox !== false){ + + $answer = [ + "title" => null, + "description" => [], + "url" => null, + "thumb" => null, + "table" => [], + "sublink" => [] + ]; + + $this->fuckhtml->load($infobox); + + $answer["title"] = + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "header", + "h5" + )[0] + ); + + $sections = + $this->fuckhtml + ->getElementsByTagName("section"); + + $i = -1; + foreach($sections as $section){ + + $this->fuckhtml->load($section); + $items = + $this->fuckhtml + ->getElementsByTagName("*"); + + $li = 1; + $pronounce = false; + foreach($items as $item){ + + switch($item["tagName"]){ + + case "h6": + + if( + isset($item["attributes"]["class"]) && + $item["attributes"]["class"] == "h6 pronunciation" + ){ + + if($pronounce){ + + break; + } + + $answer["description"][] = [ + "type" => "quote", + "value" => + $this->fuckhtml + ->getTextContent( + $item + ) + ]; + + $answer["description"][] = + [ + "type" => "audio", + "url" => "https://search.brave.com/api/rhfetch?rhtype=definitions&word={$answer["title"]}&source=ahd-5" + ]; + + $pronounce = true; + $i = $i + 2; + break; + } + + $answer["description"][] = [ + "type" => "title", + "value" => + $this->fuckhtml + ->getTextContent( + $item + ) + ]; + $i++; + break; + + case "li": + + if( + $i !== -1 && + $answer["description"][$i]["type"] == "text" + ){ + + $answer["description"][$i]["value"] .= + "\n" . $li . ". " . + $this->fuckhtml + ->getTextContent( + $item + ); + + }else{ + $answer["description"][] = [ + "type" => "text", + "value" => + $li . ". " . + $this->fuckhtml + ->getTextContent( + $item + ) + ]; + $i++; + } + $li++; + break; + + case "a": + $answer["url"] = + $this->fuckhtml + ->getTextContent( + $item["attributes"]["href"] + ); + break; + } + } + } + + $out["answer"][] = $answer; + } + + + /* + Get instant answer + */ + $this->fuckhtml->load($html); + $infobox = $this->fuckhtml->getElementById("infobox", "div"); + + if($infobox !== false){ + + $answer = [ + "title" => null, + "description" => [], + "url" => null, + "thumb" => null, + "table" => [], + "sublink" => [] + ]; + + $this->fuckhtml->load($infobox); + $div = $this->fuckhtml->getElementsByTagName("div"); + + /* + Get title + url + */ + $title = + $this->fuckhtml + ->getElementsByClassName("infobox-title", "a"); + + if(count($title) !== 0){ + + $answer["title"] = + $this->fuckhtml + ->getTextContent( + $title[0] + ); + + $answer["url"] = + $this->fuckhtml + ->getTextContent( + $title[0]["attributes"]["href"] + ); + } + + /* + Get thumbnail + */ + $thumb = $this->getimagelinkfromstyle("thumb"); + + if($thumb["url"] !== null){ + + $answer["thumb"] = $thumb["url"]; + } + + /* + Get table + */ + $title = + $this->fuckhtml + ->getElementsByClassName( + "infobox-attr-header", + "div" + ); + + $rowhtml = $infobox; + + if(count($title) >= 2){ + + $rowhtml = + explode( + $title[1]["outerHTML"], + $infobox["innerHTML"], + 2 + )[0]; + } + + $this->fuckhtml->load($rowhtml); + + $rows = + $this->fuckhtml + ->getElementsByClassName("infobox-attr", "div"); + + foreach($rows as $row){ + + if(!isset($row["innerHTML"])){ + + continue; + } + + $this->fuckhtml->load($row); + $span = + $this->fuckhtml + ->getElementsByTagName("span"); + + if(count($span) === 2){ + + $answer["table"][ + $this->fuckhtml->getTextContent($span[0]) + ] = str_replace("\n", ", ", $this->fuckhtml->getTextContent($span[1], true)); + } + } + + $this->fuckhtml->load($infobox); + + /* + Parse stackoverflow answers + */ + $code = + $this->fuckhtml + ->getElementById("codebox-answer", $div); + + if($code){ + + // this might be standalone text with no paragraphs, check for that + $author = + $this->fuckhtml + ->getElementById("author"); + + $desc_tmp = + str_replace( + $author["outerHTML"], + "", + $code["innerHTML"] + ); + + $this->fuckhtml->load($desc_tmp); + $code = + $this->fuckhtml + ->getElementsByTagName("*"); + + if(count($code) === 0){ + + $answer["description"] = + [ + [ + "type" => "text", + "value" => + $this->fuckhtml + ->getTextContent( + $desc_tmp + ) + ], + [ + "type" => "quote", + "value" => + $this->fuckhtml + ->getTextContent( + $author + ) + ] + ]; + }else{ + + $text = []; + $i = 0; + + foreach($code as $snippet){ + + switch($snippet["tagName"]){ + + case "p": + $this->fuckhtml->load($snippet["innerHTML"]); + + $codetags = + $this->fuckhtml + ->getElementsByTagName("*"); + + $tmphtml = $snippet["innerHTML"]; + + foreach($codetags as $tag){ + + if(!isset($tag["outerHTML"])){ + + continue; + } + + $tmphtml = + explode( + $tag["outerHTML"], + $tmphtml, + 2 + ); + + $value = $this->fuckhtml->getTextContent($tmphtml[0], false, false); + $this->appendtext($value, $text, $i); + + $type = null; + switch($tag["tagName"]){ + + case "code": $type = "inline_code"; break; + case "em": $type = "italic"; break; + case "blockquote": $type = "quote"; break; + default: $type = "text"; + } + + if($type !== null){ + $value = $this->fuckhtml->getTextContent($tag, false, true); + + if(trim($value) != ""){ + + if( + $i !== 0 && + $type == "title" + ){ + + $text[$i - 1]["value"] = rtrim($text[$i - 1]["value"]); + } + + $text[] = [ + "type" => $type, + "value" => $value + ]; + $i++; + } + } + + if(count($tmphtml) === 2){ + + $tmphtml = $tmphtml[1]; + }else{ + + break; + } + } + + if(is_array($tmphtml)){ + + $tmphtml = $tmphtml[0]; + } + + if(strlen($tmphtml) !== 0){ + + $value = $this->fuckhtml->getTextContent($tmphtml, false, false); + $this->appendtext($value, $text, $i); + } + break; + + case "pre": + + switch($text[$i - 1]["type"]){ + + case "text": + case "italic": + $text[$i - 1]["value"] = rtrim($text[$i - 1]["value"]); + break; + } + + $text[] = + [ + "type" => "code", + "value" => + rtrim( + $this->fuckhtml + ->getTextContent( + $snippet, + true, + false + ) + ) + ]; + $i++; + + break; + + case "ol": + $o = 0; + + $this->fuckhtml->load($snippet); + $li = + $this->fuckhtml + ->getElementsByTagName("li"); + + foreach($li as $elem){ + $o++; + + $this->appendtext( + $o . ". " . + $this->fuckhtml + ->getTextContent( + $elem + ), + $text, + $i + ); + } + break; + } + } + + if( + $i !== 0 && + $text[$i - 1]["type"] == "text" + ){ + + $text[$i - 1]["value"] = rtrim($text[$i - 1]["value"]); + } + + if($author){ + + $text[] = [ + "type" => "quote", + "value" => $this->fuckhtml->getTextContent($author) + ]; + } + + $answer["description"] = $text; + } + }else{ + + /* + Get normal description + */ + $description = + $this->fuckhtml + ->getElementsByClassName( + "mb-6", + "div" + ); + + if(count($description) !== 0){ + + $description = + [ + [ + "type" => "text", + "value" => + $this->titledots( + preg_replace( + '/ Wikipedia$/', + "", + $this->fuckhtml + ->getTextContent( + $description[0] + ) + ) + ) + ] + ]; + + $ratings = + $this->fuckhtml + ->getElementById("ratings"); + + if($ratings){ + + $this->fuckhtml->load($ratings); + + $ratings = + $this->fuckhtml + ->getElementsByClassName( + "flex-hcenter mb-10", + "div" + ); + + $description[] = [ + "type" => "title", + "value" => "Ratings" + ]; + + foreach($ratings as $rating){ + + $this->fuckhtml->load($rating); + + $num = + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "r-num", + "div" + )[0] + ); + + $href = + $this->fuckhtml + ->getElementsByClassName( + "mr-10", + "a" + )[0]; + + $votes = + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "text-sm", + "span" + )[0] + ); + + $c = count($description) - 1; + + if( + $c !== -1 && + $description[$c]["type"] == "text" + ){ + + $description[$c]["value"] .= $num . " "; + }else{ + + $description[] = [ + "type" => "text", + "value" => $num . " " + ]; + } + + $description[] = [ + "type" => "link", + "value" => $this->fuckhtml->getTextContent($href), + "url" => $this->fuckhtml->getTextContent($href["attributes"]["href"]) + ]; + + $description[] = [ + "type" => "text", + "value" => " (" . $votes . ")\n" + ]; + } + } + + $answer["description"] = $description; + } + } + + /* + Get sublinks + */ + $this->fuckhtml->load($infobox); + + $profiles = + $this->fuckhtml + ->getElementById("profiles"); + + if($profiles){ + $profiles = + $this->fuckhtml + ->getElementsByClassName( + "chip", + "a" + ); + + foreach($profiles as $profile){ + + $name = $this->fuckhtml->getTextContent($profile["attributes"]["title"]); + + if(strtolower($name) == "steampowered"){ + + $name = "Steam"; + } + + $answer["sublink"][$name] = + $this->fuckhtml->getTextContent($profile["attributes"]["href"]); + } + } + + $actors = + $this->fuckhtml + ->getElementById("panel-movie-cast"); + + if($actors){ + + $this->fuckhtml->load($actors); + + $actors = + $this->fuckhtml + ->getElementsByClassName("card"); + + $answer["description"][] = [ + "type" => "title", + "value" => "Cast" + ]; + + foreach($actors as $actor){ + + $this->fuckhtml->load($actor); + + $answer["description"][] = [ + "type" => "text", + "value" => + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName("card-body") + [0] + ) + ]; + + $answer["description"][] = [ + "type" => "image", + "url" => $this->getimagelinkfromstyle("person-thumb")["url"] + ]; + } + } + + $out["answer"][] = $answer; + } + + /* + Get actor standalone thingy + */ + $this->fuckhtml->load($resulthtml); + $actors = + $this->fuckhtml + ->getElementById("predicate-entity"); + + if($actors){ + + $this->fuckhtml->load($actors); + + $cards = + $this->fuckhtml + ->getElementsByClassName("card"); + + $url = + $this->fuckhtml + ->getElementsByClassName( + "disclaimer", + "div" + )[0]; + + $this->fuckhtml->load($url); + + $url = + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByTagName("a") + [0] + ["attributes"] + ["href"] + ); + + $this->fuckhtml->load($actors); + + $answer = [ + "title" => + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "entity", + "span" + )[0] + ) . " (Cast)", + "description" => [], + "url" => $url, + "sublink" => [], + "thumb" => null, + "table" => [] + ]; + + foreach($cards as $card){ + + $this->fuckhtml->load($card); + + $answer["description"][] = [ + "type" => "title", + "value" => + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "title" + )[0] + ) + ]; + + $answer["description"][] = [ + "type" => "text", + "value" => + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "text-xs desc" + )[0] + ) + ]; + + $answer["description"][] = [ + "type" => "image", + "url" => $this->getimagelinkfromstyle("img-bg")["url"] + ]; + } + + $out["answer"][] = $answer; + } + + return $out; + } + + public function news($get){ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $nsfw = $get["nsfw"]; + $country = $get["country"]; + + if(strlen($search) > 2048){ + + throw new Exception("Search query is too long!"); + } + /* + $handle = fopen("scraper/brave-news.html", "r"); + $html = fread($handle, filesize("scraper/brave-news.html")); + fclose($handle);*/ + try{ + $html = + $this->get( + "https://search.brave.com/news", + [ + "q" => $search + ], + $nsfw, + $country + ); + + }catch(Exception $error){ + + throw new Exception("Could not fetch search page"); + } + + $out = [ + "status" => "ok", + "npt" => null, + "news" => [] + ]; + + // load html + $this->fuckhtml->load($html); + + $news = + $this->fuckhtml + ->getElementsByClassName( + "snippet inline gap-standard", + "div" + ); + + foreach($news as $article){ + + $data = [ + "title" => null, + "author" => null, + "description" => null, + "date" => null, + "thumb" => + [ + "url" => null, + "ratio" => null + ], + "url" => null + ]; + + $this->fuckhtml->load($article); + $elems = + $this->fuckhtml + ->getElementsByTagName("*"); + + // get title + $data["title"] = + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "snippet-title", + $elems + ) + [0] + ["innerHTML"] + ); + + // get description + $data["description"] = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "snippet-description", + $elems + ) + [0] + ["innerHTML"] + ) + ); + + // get date + $date = + explode( + "•", + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "snippet-url", + $elems + )[0] + ) + ); + + if( + count($date) !== 1 && + trim($date[1]) != "" + ){ + + $data["date"] = + strtotime( + $date[1] + ); + } + + // get URL + $data["url"] = + $this->fuckhtml->getTextContent( + $this->unshiturl( + $this->fuckhtml + ->getElementsByClassName( + "result-header", + $elems + ) + [0] + ["attributes"] + ["href"] + ) + ); + + // get thumbnail + $thumb = + $this->fuckhtml + ->getElementsByTagName( + "img" + ); + + if( + count($thumb) === 2 && + trim( + $thumb[1] + ["attributes"] + ["src"] + ) != "" + ){ + + $data["thumb"] = [ + "url" => + $this->fuckhtml->getTextContent( + $this->unshiturl( + $thumb[1] + ["attributes"] + ["src"] + ) + ), + "ratio" => "16:9" + ]; + } + + $out["news"][] = $data; + } + + return $out; + } + + /* + public function bypasscaptcha($html, $nsfw, $country){ + + // @TODO figure out why I still cant go trough + // the captcha wall even after breaking it + + try{ + $html = + $this->get( + "https://search.brave.com/goggles", + [ + "q" => "site:dailymotion.com my bloody valentine" + ], + $nsfw, + $country + ); + + }catch(Exception $error){ + + throw new Exception("Could not fetch html"); + } + + // Bypass brave search captcha + // this captcha only appears on the goggles page + preg_match( + '/this\.img\.src = "(.*)"/', + $html, + $image + ); + + $image = + base64_decode( + explode( + "data:image/png;base64,", + $image[1] + )[1] + ); + + $im = new Imagick(); + $im->readImageBlob($image); + + $im->blurImage(20, 20); + $im->posterizeImage(2, imagick::IMGTYPE_COLORSEPARATION); + + // if we encounter a white line thats longer than 45px + // we found the circle position + $iterator = $im->getPixelRegionIterator(0, 77, 310, 1); + + $found = null; + foreach( + $iterator as $row + ){ + + $whitecount = 0; + $count = 0; + + foreach($row as $pixel){ + + if($pixel->getColor()["r"] === 255){ + + $whitecount++; + $pixel->setColor("rgba(255,0,0,0)"); + + if($whitecount === 45){ + + $found = $count - 45; + break 2; + } + }else{ + + $whitecount = 0; + } + + $count++; + $iterator->syncIterator(); + } + } + + $found = $found + 10; + + //header("Content-Type: image/png"); + //echo $im; + //die(); + + if($found === null){ + + throw new Exception("Could not bypass captcha"); + } + + preg_match( + '/data="{"captcha_id":"([0-9A-z-]+)"}"/', + $html, + $key + ); + + $key = $key[1]; + // we bypassed captcha, send POST data + $order = + $this->get( + "https://search.brave.com/api/captcha?brave=0&captcha_id={$key}", + [ + "solution" => (string)$found + ], + $nsfw, + $country, + true + ); + + $order = json_decode($order, true)["orderId"]; + + $orderpayload = + $this->get( + "https://search.brave.com/api/rewards/v1/orders/{$order}", + [], + $nsfw, + $country + ); + + $orderpayload = json_decode($orderpayload, true); + + $creds = + $this->get( + "https://search.brave.com/api/rewards/v1/orders/{$order}/credentials", + [ + "itemId" => $orderpayload["items"][0]["id"], + "blindedCreds" => [ + "fuYAVcB/m7BU66vf3wkNGxJCSaRhshB9o+8km3F1h2c=", + "uswvcWJuPK/1qFlVdzBP3eQd0+V1EQgfAtnEoMIK+Uk=", + "fJWKGLBxl3Gyn4n9FjTLq1PjupfABT7Ni8MeB+iGzUs=", + "Aq9enJ/VZP9GxQIza3n65ZK7xQhY4VwDxv53BCb/Txg=", + "FMJA9eSLHq71K+Pcwgm4gIQOmdR/6KMy5cMgXhpd5Ro=", + "2NVhIAbvI317SP9/xXbVe/U57eWgvHyqVbHL/5+Gdmw=", + "6mpjsjSCmYEzK2xlbL8DI2P4LuhWUOxjTLvsTAL9l24=", + "kAn4wuHvIlKWhfuFfPTSfD4tZ5le9t7/61YbdEc/L3k=", + "BjjUyG16aTfd1c0h4oBzgQQOekrH1f+a5CmcXqMPTR4=", + "SBNgpCt4/V44yaQTfh+D027Yv1GJFHkjUEpPw6rAwRI=", + "XDENAtdQ7PyYx+Qx1wQGQtDWgg8WpIMgWGmd4RDOVWE=", + "tF7rB4sqamsiUk3K7fojdQSI0Q6iip72yKyhnvg/bC0=", + "VsAqflirAd/u4VsLdfRS2UvnH24ZNkFh6YN3DctLjzQ=", + "MntLbXkoI0LdcisCbNazmooiHXJyX91L1KERDAu1JRU=", + "TH6Zs8JBvFDbTDWgKbfGE4M5/cSwCtHD8ms5Y/U8zHQ=", + "jsZg0Z+qDPHymrbhdnesodhLNJ26QdunyMko1aVe4So=", + "rpKsyj6/vdnuMgLI2BApeijtGq9g5USRDL0w6X2bnlQ=", + "vCzliGT8A9vcLXj2sFf2kavOuYw69d70NpfgA22B4lI=", + "7OWoxSCtYXWcaBSifF7AXNBif/sjcuO0IelzXG/3PFk=", + "iiXtByNlT6nDMN9De5B58Jl8J0p6LCjnZ9aS3w2FEQU=", + "zDhd7gsJ4h4JkDeGK0Y0mfFd8IBdkLhMOANzwO+4Dig=", + "qANZ+AikwFReEA61JF009d/c3IHM/aSfIYwljckhJWE=", + "nNC30pDLxtXvUr+WDwfDSrAInNBpfSZkPsV2JlpheWI=", + "kGXE1pkt25P71kdJzmKIg4+yMR1VA5wNmbpBb/FhJQ8=", + "aLqPsY1Qiz2UCa2Jx3YNNt8r4JINMphks/43EiyZfXU=", + "bHGYZoQARZEM5LdFF6B74PkRqNd9EKxzuTvGYxjq+hk=", + "JOsYQjfE/9Y1u29hR+GvEkNyxUI8blgLhX1iJI/aGRQ=", + "yKjHjH5j600TJD/3WPsA1N3OmItDLifdjlysq4H6NV0=", + "9lTnUbsPp7BJ7XVN5/T4yGfzD9DJdqWB7xk72s19MAA=", + "5KHG8iY45em7zDhO/HlI0ydcZ0Ubn+XSyjifMmy7qXM=" + ] + ], + $nsfw, + $country, + true + ); + + var_dump($creds); + + sleep(2); + $test = + $this->get( + "https://search.brave.com/api/rewards/v1/orders/{$order}/credentials", + [], + $nsfw, + $country + ); + + var_dump($test); + + $html = + $this->get( + "https://search.brave.com/goggles", + [ + "q" => "site:dailymotion.com my bloody valentine" + ], + $nsfw, + $country, + false, + "__Secure-sku#brave-search-captcha=eyJ0eXBlIjoic2luZ2xlLXVzZSIsInZlcnNpb24iOjEsInNrdSI6ImJyYXZlLXNlYXJjaC1jYXB0Y2hhIiwicHJlc2VudGF0aW9uIjoiZXlKcGMzTjFaWElpT2lKaWNtRjJaUzVqYjIwL2MydDFQV0p5WVhabExYTmxZWEpqYUMxallYQjBZMmhoSWl3aWMybG5ibUYwZFhKbElqb2lNRzl0VDBneWQxZ3dTazkzU0VFMVJ6QTJaR1V5WjFOQ1dDdGhSM3B2Y2xsTVQwVTJZVVJtTUc5a1IweG1Wa3RhZEd0cU4xbHdia3BPT0VOVGNGbE5lVWR2YmpGRlNTOUhhMlZYU1RWNGQxTjJPWGxJTTNjOVBTSXNJblFpT2lKWlJWWldaVzR5TTJwQ01tSnZkakJ2U1hGNGJtSndUMGxEUW5Kd1drRjBRbWQxVnpoRlNURTNVREY2UVRaQlpUTXJSVGRFYm5NeVFqUmhka0pGYTFWM2FGY3JWRVZJVjNWcE9TdFllRU1yYlVSTVkyMTBRVDA5SW4wPSJ9" + ); + + var_dump($html); + }*/ + + private function appendtext($payload, &$text, &$index){ + + if(trim($payload) == ""){ + + return; + } + + if( + $index !== 0 && + $text[$index - 1]["type"] == "text" + ){ + + $text[$index - 1]["value"] .= "\n\n" . preg_replace('/ $/', " ", $payload); + }else{ + + $text[] = [ + "type" => "text", + "value" => preg_replace('/ $/', " ", $payload) + ]; + $index++; + } + } + + private function tablesublink($html_collection, &$data){ + + foreach($html_collection as $html){ + + $html["innerHTML"] = preg_replace( + '/<style>[\S\s]*<\/style>/i', + "", + $html["innerHTML"] + ); + + $html = + explode( + ":", + $this->fuckhtml->getTextContent($html), + 2 + ); + + if(count($html) === 1){ + + $html = ["Rating", $html[0]]; + } + + $data["table"][trim($html[0])] = trim($html[1]); + } + } + + private function getimagelinkfromstyle($thumb){ + + $thumb = + $this->fuckhtml + ->getElementsByClassName( + $thumb, + "div" + ); + + if(count($thumb) === 0){ + + return [ + "url" => null, + "ratio" => null + ]; + } + + $thumb = $thumb[0]["attributes"]["style"]; + + preg_match( + '/background-image: ?url\((\'[^\']+\'|"[^"]+"|[^\)]+)\)/', + $thumb, + $thumb + ); + + $url = $this->fuckhtml->getTextContent($this->unshiturl(trim($thumb[1], '"\' '))); + + if(parse_url($url, PHP_URL_HOST) == "cdn.search.brave.com"){ + + return [ + "url" => null, + "ratio" => null + ]; + } + + return [ + "url" => $url, + "ratio" => "16:9" + ]; + } + + private function limitstrlen($text){ + + return explode("\n", wordwrap($text, 300, "\n"))[0]; + } + + private function limitwhitespace($text){ + + return + preg_replace( + '/[\s]+/', + " ", + $text + ); + } + + private function titledots($title){ + + $substr = substr($title, -3); + + if( + $substr == "..." || + $substr == "…" + ){ + + return trim(substr($title, 0, -3)); + } + + return trim($title); + } + + private function unshiturl($url){ + + // https://imgs.search.brave.com/XFnbR8Sl7ge82MBDEH7ju0UHImRovMVmQ2qnDvgNTuA/rs:fit:844:225:1/g:ce/aHR0cHM6Ly90c2U0/Lm1tLmJpbmcubmV0/L3RoP2lkPU9JUC54/UWotQXU5N2ozVndT/RDJnNG9BNVhnSGFF/SyZwaWQ9QXBp.jpeg + + $tmp = explode("aHR0", $url); + + if(count($tmp) !== 2){ + + // nothing to do + return $url; + } + + return + base64_decode( + "aHR0" . + str_replace(["/", "_"], ["", "/"], + explode( + ".", + $tmp[1] + )[0] + ) + ); + } +} diff --git a/scraper/ddg.php b/scraper/ddg.php new file mode 100644 index 0000000..c9c28af --- /dev/null +++ b/scraper/ddg.php @@ -0,0 +1,2722 @@ +<?php + +class ddg{ + + public function __construct(){ + + include "lib/nextpage.php"; + $this->nextpage = new nextpage("ddg"); + } + + /* + curl functions + */ + private const req_web = 0; + private const req_xhr = 1; + + private function get($url, $get = [], $reqtype = self::req_web){ + + $curlproc = curl_init(); + + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + + curl_setopt($curlproc, CURLOPT_URL, $url); + + switch($reqtype){ + case self::req_web: + $headers = + ["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Encoding: gzip", + "Accept-Language: en-US,en;q=0.5", + "DNT: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: cross-site", + "Upgrade-Insecure-Requests: 1"]; + break; + + case self::req_xhr: + $headers = + ["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0", + "Accept: */*", + "Accept-Encoding: gzip", + "Accept-Language: en-US,en;q=0.5", + "Connection: keep-alive", + "Referer: https://duckduckgo.com/", + "X-Requested-With: XMLHttpRequest", + "DNT: 1", + "Sec-Fetch-Dest: script", + "Sec-Fetch-Mode: no-cors", + "Sec-Fetch-Site: same-site"]; + break; + } + + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + public function getfilters($pagetype){ + + switch($pagetype){ + + case "web": + return + [ + "country" => [ + "display" => "Country", + "option" => [ + "any" => "All Regions", + "ar-es" => "Argentina", + "au-en" => "Australia", + "at-de" => "Austria", + "be-fr" => "Belgium (fr)", + "be-nl" => "Belgium (nl)", + "br-pt" => "Brazil", + "bg-bg" => "Bulgaria", + "ca-en" => "Canada (en)", + "ca-fr" => "Canada (fr)", + "ct-ca" => "Catalonia", + "cl-es" => "Chile", + "cn-zh" => "China", + "co-es" => "Colombia", + "hr-hr" => "Croatia", + "cz-cs" => "Czech Republic", + "dk-da" => "Denmark", + "ee-et" => "Estonia", + "fi-fi" => "Finland", + "fr-fr" => "France", + "de-de" => "Germany", + "gr-el" => "Greece", + "hk-tzh" => "Hong Kong", + "hu-hu" => "Hungary", + "in-en" => "India (en)", + "id-en" => "Indonesia (en)", + "ie-en" => "Ireland", + "il-en" => "Israel (en)", + "it-it" => "Italy", + "jp-jp" => "Japan", + "kr-kr" => "Korea", + "lv-lv" => "Latvia", + "lt-lt" => "Lithuania", + "my-en" => "Malaysia (en)", + "mx-es" => "Mexico", + "nl-nl" => "Netherlands", + "nz-en" => "New Zealand", + "no-no" => "Norway", + "pk-en" => "Pakistan (en)", + "pe-es" => "Peru", + "ph-en" => "Philippines (en)", + "pl-pl" => "Poland", + "pt-pt" => "Portugal", + "ro-ro" => "Romania", + "ru-ru" => "Russia", + "xa-ar" => "Saudi Arabia", + "sg-en" => "Singapore", + "sk-sk" => "Slovakia", + "sl-sl" => "Slovenia", + "za-en" => "South Africa", + "es-ca" => "Spain (ca)", + "es-es" => "Spain (es)", + "se-sv" => "Sweden", + "ch-de" => "Switzerland (de)", + "ch-fr" => "Switzerland (fr)", + "tw-tzh" => "Taiwan", + "th-en" => "Thailand (en)", + "tr-tr" => "Turkey", + "us-en" => "US (English)", + "us-es" => "US (Spanish)", + "ua-uk" => "Ukraine", + "uk-en" => "United Kingdom", + "vn-en" => "Vietnam (en)" + ] + ], + "nsfw" => [ + "display" => "NSFW", + "option" => [ + "yes" => "Yes", + "maybe" => "Maybe", + "no" => "No" + ] + ], + "newer" => [ + "display" => "Newer than", + "option" => "_DATE" + ], + "older" => [ + "display" => "Older than", + "option" => "_DATE" + ], + "extendedsearch" => [ + // undefined display, so it wont show in frontend + "option" => [ + "yes" => "Yes", + "no" => "No" + ] + ] + ]; + break; + + case "images": + return + [ + "country" => [ + "display" => "Country", + "option" => [ + "us-en" => "US (English)", + "ar-es" => "Argentina", + "au-en" => "Australia", + "at-de" => "Austria", + "be-fr" => "Belgium (fr)", + "be-nl" => "Belgium (nl)", + "br-pt" => "Brazil", + "bg-bg" => "Bulgaria", + "ca-en" => "Canada (en)", + "ca-fr" => "Canada (fr)", + "ct-ca" => "Catalonia", + "cl-es" => "Chile", + "cn-zh" => "China", + "co-es" => "Colombia", + "hr-hr" => "Croatia", + "cz-cs" => "Czech Republic", + "dk-da" => "Denmark", + "ee-et" => "Estonia", + "fi-fi" => "Finland", + "fr-fr" => "France", + "de-de" => "Germany", + "gr-el" => "Greece", + "hk-tzh" => "Hong Kong", + "hu-hu" => "Hungary", + "in-en" => "India (en)", + "id-en" => "Indonesia (en)", + "ie-en" => "Ireland", + "il-en" => "Israel (en)", + "it-it" => "Italy", + "jp-jp" => "Japan", + "kr-kr" => "Korea", + "lv-lv" => "Latvia", + "lt-lt" => "Lithuania", + "my-en" => "Malaysia (en)", + "mx-es" => "Mexico", + "nl-nl" => "Netherlands", + "nz-en" => "New Zealand", + "no-no" => "Norway", + "pk-en" => "Pakistan (en)", + "pe-es" => "Peru", + "ph-en" => "Philippines (en)", + "pl-pl" => "Poland", + "pt-pt" => "Portugal", + "ro-ro" => "Romania", + "ru-ru" => "Russia", + "xa-ar" => "Saudi Arabia", + "sg-en" => "Singapore", + "sk-sk" => "Slovakia", + "sl-sl" => "Slovenia", + "za-en" => "South Africa", + "es-ca" => "Spain (ca)", + "es-es" => "Spain (es)", + "se-sv" => "Sweden", + "ch-de" => "Switzerland (de)", + "ch-fr" => "Switzerland (fr)", + "tw-tzh" => "Taiwan", + "th-en" => "Thailand (en)", + "tr-tr" => "Turkey", + "us-es" => "US (Spanish)", + "ua-uk" => "Ukraine", + "uk-en" => "United Kingdom", + "vn-en" => "Vietnam (en)" + ] + ], + "nsfw" => [ + "display" => "NSFW", + "option" => [ + "yes" => "Yes", + "no" => "No" + ] + ], + "date" => [ + "display" => "Time posted", + "option" => [ + "any" => "Any time", + "Day" => "Past day", + "Week" => "Past week", + "Month" => "Past month" + ] + ], + "size" => [ + "display" => "Size", + "option" => [ + "any" => "Any size", + "Small" => "Small", + "Medium" => "Medium", + "Large" => "Large", + "Wallpaper" => "Wallpaper" + ] + ], + "color" => [ + "display" => "Colors", + "option" => [ + "any" => "All colors", + "Monochrome" => "Black and white", + "Red" => "Red", + "Orange" => "Orange", + "Yellow" => "Yellow", + "Green" => "Green", + "Blue" => "Blue", + "Purple" => "Purple", + "Pink" => "Pink", + "Brown" => "Brown", + "Black" => "Black", + "Gray" => "Gray", + "Teal" => "Teal", + "White" => "White" + ] + ], + "type" => [ + "display" => "Type", + "option" => [ + "any" => "All types", + "photo" => "Photograph", + "clipart" => "Clipart", + "gif" => "Animated GIF", + "transparent" => "Transparent" + ] + ], + "layout" => [ + "display" => "Layout", + "option" => [ + "any" => "All layouts", + "Square" => "Square", + "Tall" => "Tall", + "Wide" => "Wide" + ] + ], + "license" => [ + "display" => "License", + "option" => [ + "any" => "All licenses", // blame ddg for this + "Any" => "All Creative Commons", + "Public" => "Public domain", + "Share" => "Free to Share and Use", + "ShareCommercially" => "Free to Share and Use Commercially", + "Modify" => "Free to Modify, Share, and Use", + "ModifyCommercially" => "Free to Modify, Share, and Use Commercially" + ] + ] + ]; + break; + + case "videos": + return + [ + "country" => [ + "display" => "Country", + "option" => [ + "us-en" => "US (English)", + "ar-es" => "Argentina", + "au-en" => "Australia", + "at-de" => "Austria", + "be-fr" => "Belgium (fr)", + "be-nl" => "Belgium (nl)", + "br-pt" => "Brazil", + "bg-bg" => "Bulgaria", + "ca-en" => "Canada (en)", + "ca-fr" => "Canada (fr)", + "ct-ca" => "Catalonia", + "cl-es" => "Chile", + "cn-zh" => "China", + "co-es" => "Colombia", + "hr-hr" => "Croatia", + "cz-cs" => "Czech Republic", + "dk-da" => "Denmark", + "ee-et" => "Estonia", + "fi-fi" => "Finland", + "fr-fr" => "France", + "de-de" => "Germany", + "gr-el" => "Greece", + "hk-tzh" => "Hong Kong", + "hu-hu" => "Hungary", + "in-en" => "India (en)", + "id-en" => "Indonesia (en)", + "ie-en" => "Ireland", + "il-en" => "Israel (en)", + "it-it" => "Italy", + "jp-jp" => "Japan", + "kr-kr" => "Korea", + "lv-lv" => "Latvia", + "lt-lt" => "Lithuania", + "my-en" => "Malaysia (en)", + "mx-es" => "Mexico", + "nl-nl" => "Netherlands", + "nz-en" => "New Zealand", + "no-no" => "Norway", + "pk-en" => "Pakistan (en)", + "pe-es" => "Peru", + "ph-en" => "Philippines (en)", + "pl-pl" => "Poland", + "pt-pt" => "Portugal", + "ro-ro" => "Romania", + "ru-ru" => "Russia", + "xa-ar" => "Saudi Arabia", + "sg-en" => "Singapore", + "sk-sk" => "Slovakia", + "sl-sl" => "Slovenia", + "za-en" => "South Africa", + "es-ca" => "Spain (ca)", + "es-es" => "Spain (es)", + "se-sv" => "Sweden", + "ch-de" => "Switzerland (de)", + "ch-fr" => "Switzerland (fr)", + "tw-tzh" => "Taiwan", + "th-en" => "Thailand (en)", + "tr-tr" => "Turkey", + "us-en" => "US (English)", + "us-es" => "US (Spanish)", + "ua-uk" => "Ukraine", + "uk-en" => "United Kingdom", + "vn-en" => "Vietnam (en)" + ] + ], + "nsfw" => [ + "display" => "NSFW", + "option" => [ + "yes" => "Yes", + "no" => "No" + ] + ], + "date" => [ + "display" => "Time fetched", + "option" => [ + "any" => "Any time", + "d" => "Past day", + "w" => "Past week", + "m" => "Past month" + ] + ], + "resolution" => [ //videoDefinition + "display" => "Resolution", + "option" => [ + "any" => "Any resolution", + "high" => "High definition", + "standard" => "Standard definition" + ] + ], + "duration" => [ // videoDuration + "display" => "Duration", + "option" => [ + "any" => "Any duration", + "short" => "Short (>5min)", + "medium" => "Medium (5-20min)", + "long" => "Long (<20min)" + ] + ], + "license" => [ + "display" => "License", + "option" => [ + "any" => "Any license", + "creativeCommon" => "Creative Commons", + "youtube" => "YouTube Standard" + ] + ] + ]; + break; + + case "news": + return + [ + "country" => [ + "display" => "Country", + "option" => [ + "us-en" => "US (English)", + "ar-es" => "Argentina", + "au-en" => "Australia", + "at-de" => "Austria", + "be-fr" => "Belgium (fr)", + "be-nl" => "Belgium (nl)", + "br-pt" => "Brazil", + "bg-bg" => "Bulgaria", + "ca-en" => "Canada (en)", + "ca-fr" => "Canada (fr)", + "ct-ca" => "Catalonia", + "cl-es" => "Chile", + "cn-zh" => "China", + "co-es" => "Colombia", + "hr-hr" => "Croatia", + "cz-cs" => "Czech Republic", + "dk-da" => "Denmark", + "ee-et" => "Estonia", + "fi-fi" => "Finland", + "fr-fr" => "France", + "de-de" => "Germany", + "gr-el" => "Greece", + "hk-tzh" => "Hong Kong", + "hu-hu" => "Hungary", + "in-en" => "India (en)", + "id-en" => "Indonesia (en)", + "ie-en" => "Ireland", + "il-en" => "Israel (en)", + "it-it" => "Italy", + "jp-jp" => "Japan", + "kr-kr" => "Korea", + "lv-lv" => "Latvia", + "lt-lt" => "Lithuania", + "my-en" => "Malaysia (en)", + "mx-es" => "Mexico", + "nl-nl" => "Netherlands", + "nz-en" => "New Zealand", + "no-no" => "Norway", + "pk-en" => "Pakistan (en)", + "pe-es" => "Peru", + "ph-en" => "Philippines (en)", + "pl-pl" => "Poland", + "pt-pt" => "Portugal", + "ro-ro" => "Romania", + "ru-ru" => "Russia", + "xa-ar" => "Saudi Arabia", + "sg-en" => "Singapore", + "sk-sk" => "Slovakia", + "sl-sl" => "Slovenia", + "za-en" => "South Africa", + "es-ca" => "Spain (ca)", + "es-es" => "Spain (es)", + "se-sv" => "Sweden", + "ch-de" => "Switzerland (de)", + "ch-fr" => "Switzerland (fr)", + "tw-tzh" => "Taiwan", + "th-en" => "Thailand (en)", + "tr-tr" => "Turkey", + "us-en" => "US (English)", + "us-es" => "US (Spanish)", + "ua-uk" => "Ukraine", + "uk-en" => "United Kingdom", + "vn-en" => "Vietnam (en)" + ] + ], + "nsfw" => [ + "display" => "NSFW", + "option" => [ + "yes" => "Yes", + "maybe" => "Maybe", + "no" => "No" + ] + ], + "date" => [ + "display" => "Time posted", + "option" => [ + "any" => "Any time", + "d" => "Past day", + "w" => "Past week", + "m" => "Past month" + ] + ] + ]; + break; + + default: + return []; + break; + } + } + + public function web($get){ + + if($get["npt"]){ + + $jsgrep = $this->nextpage->get($get["npt"], "web"); + + $extendedsearch = false; + $inithtml = ""; + + }else{ + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $country = $get["country"]; + $nsfw = $get["nsfw"]; + $older = $get["older"]; + $newer = $get["newer"]; + $extendedsearch = $get["extendedsearch"] == "yes" ? true : false; + + // generate filters + $get_filters = [ + "q" => $search, + "kz" => "1" // force instant answers + ]; + + if($country == "any"){ + + $get_filters["kl"] = "wt-wt"; + }else{ + + $get_filters["kl"] = $country; + } + + switch($nsfw){ + + case "yes": $get_filters["kp"] = "-2"; break; + case "maybe": $get_filters["kp"] = "-1"; break; + case "no": $get_filters["kp"] = "1"; break; + } + + $df = true; + + if($newer === false){ + + if($older !== false){ + + $start = 36000; + $end = $older; + }else{ + + $df = false; + } + }else{ + + $start = $newer; + + if($older !== false){ + + $end = $older; + }else{ + + $end = time(); + } + } + + if($df === true){ + $get_filters["df"] = date("Y-m-d", $start) . ".." . date("Y-m-d", $end); + } + + /* + Get html + */ + // https://duckduckgo.com/?q=minecraft&kz=1&k1=-1&kp=-2 + try{ + $inithtml = $this->get( + "https://duckduckgo.com/", + $get_filters + ); + }catch(Exception $e){ + + throw new Exception("Failed to get html"); + } + + preg_match( + '/DDG\.deep\.initialize\(\'(.*)\',/U', + $inithtml, + $jsgrep + ); + + if(!isset($jsgrep[1])){ + + throw new Exception("Failed to get d.js URL"); + } + + $jsgrep = $jsgrep[1]; + } + + // get javascript + try{ + + $js = $this->get( + "https://links.duckduckgo.com" . $jsgrep, + [], + ddg::req_xhr + ); + }catch(Exception $e){ + + throw new Exception("Failed to fetch d.js"); + } + + // initialize api response array + $out = [ + "status" => "ok", + "spelling" => [ + "type" => "no_correction", + "using" => null, + "correction" => null + ], + "npt" => null, + "answer" => [], + "web" => [], + "image" => [], + "video" => [], + "news" => [], + "related" => [] + ]; + + /* + Additional requests + */ + + if($extendedsearch){ + + /* + Check for worknik results + */ + preg_match( + '/nrj\(\'\/js\/spice\/dictionary\/definition\/([^\']+)\'\)/', + $js, + $wordnik + ); + + if(isset($wordnik[1])){ + + try{ + + $wordnik = $wordnik[1]; + + // get definition + $wordnikjs = $this->get( + "https://duckduckgo.com/js/spice/dictionary/definition/" . $wordnik, + [], + ddg::req_xhr + ); + + preg_match( + '/ddg_spice_dictionary_definition\(\n?(\[{[\S\s]*}])/', + $wordnikjs, + $wordnikjson + ); + + if(isset($wordnikjson[1])){ + + $wordnikjson = json_decode($wordnikjson[1], true); + + $out["answer"][0] = [ + "title" => urldecode($wordnik), + "description" => [], + "url" => "https://www.wordnik.com/words/" . $wordnik, + "thumb" => null, + "table" => [], + "sublink" => [] + ]; + + $partofspeech = false; + $wastext = false; + $textindent = 1; + + // get audio + + $wordnikaudio_json = + json_decode( + $this->get( + "https://duckduckgo.com/js/spice/dictionary/audio/" . $wordnik, + [], + ddg::req_xhr + ), + true + ); + + if(isset($wordnikaudio_json[0]["id"])){ + + usort($wordnikaudio_json, function($a, $b){ + + return $a["id"] < $b["id"]; + }); + + $out["answer"][0]["description"][] = [ + "type" => "audio", + "url" => $wordnikaudio_json[0]["fileUrl"] + ]; + } + + $collection = []; + $e[] = []; + + foreach($wordnikjson as $data){ + + if(!isset($data["partOfSpeech"])){ + + continue; + } + + if(isset($data["text"])){ + + if(!isset($collection[$data["partOfSpeech"]])){ + + $collection[$data["partOfSpeech"]] = []; + $c = 0; + }else{ + $c = count($collection[$data["partOfSpeech"]]); + } + + if(!isset($e[$data["partOfSpeech"]])){ + + $e[$data["partOfSpeech"]] = 0; + } + + $e[$data["partOfSpeech"]]++; + $text = $e[$data["partOfSpeech"]] . ". " . $this->unescapehtml(strip_tags($data["text"])); + + $syn = false; + if( + isset($data["relatedWords"]) && + count($data["relatedWords"]) !== 0 + ){ + + $syn = " ("; + + $u = 0; + foreach($data["relatedWords"] as $related){ + + $syn .= ucfirst($related["relationshipType"]) . ": "; + + $c = count($related["words"]); + $b = 0; + foreach($related["words"] as $word){ + + $syn .= trim($this->unescapehtml(strip_tags($word))); + + $b++; + if($b !== $c){ + + $syn .= ", "; + } + } + + $u++; + if($u !== count($data["relatedWords"])){ + + $syn .= ". "; + } + } + + $syn .= ")"; + } + + if( + $c !== 0 && + $collection[$data["partOfSpeech"]][$c - 1]["type"] == "text" + ){ + $collection[$data["partOfSpeech"]][$c - 1]["value"] .= + "\n" . $text; + + }else{ + + if( + $c !== 0 && + ( + $collection[$data["partOfSpeech"]][$c - 1]["type"] == "text" || + $collection[$data["partOfSpeech"]][$c - 1]["type"] == "italic" + ) + ){ + + $text = "\n" . $text; + } + + $collection[$data["partOfSpeech"]][] = + [ + "type" => "text", + "value" => $text + ]; + } + + if($syn){ + + $collection[$data["partOfSpeech"]][] = [ + "type" => "italic", + "value" => $syn + ]; + } + + if(isset($data["exampleUses"])){ + + foreach($data["exampleUses"] as $use){ + + $collection[$data["partOfSpeech"]][] = [ + "type" => "quote", + "value" => $this->unescapehtml(strip_tags($use["text"])) + ]; + } + } + + if(isset($data["citations"])){ + + foreach($data["citations"] as $citation){ + + if(!isset($citation["cite"])){ + + continue; + } + + $value = $this->unescapehtml(strip_tags($citation["cite"])); + + if( + isset($citation["source"]) && + trim($citation["source"]) != "" + ){ + $value .= " - " . $this->unescapehtml(strip_tags($citation["source"])); + } + + $collection[$data["partOfSpeech"]][] = [ + "type" => "quote", + "value" => $value + ]; + } + } + } + } + + foreach($collection as $key => $items){ + + $out["answer"][0]["description"][] = + [ + "type" => "title", + "value" => $key + ]; + + $out["answer"][0]["description"] = + array_merge($out["answer"][0]["description"], $items); + } + } + + }catch(Exception $e){ + + // do nothing + } + } + + unset($wordnik); + + /* + Check for stackoverflow answers + */ + + // /a.js?p=1&src_id=stack_overflow&from=nlp_qa&id=3390396,2559318&q=how%20can%20i%20check%20for%20undefined%20in%20javascript&s=stackoverflow.com&tl=How%20can%20I%20check%20for%20%22undefined%22%20in%20JavaScript%3F%20%2D%20Stack%20Overflow + // /a.js?p=1&src_id=arqade&from=nlp_qa&id=370293,375682&q=what%20is%20the%20difference%20between%20at%20and%20positioned%20in%20execute&s=gaming.stackexchange.com&tl=minecraft%20java%20edition%20minecraft%20commands%20%2D%20What%20is%20the%20difference + // /a.js?p=1&src_id=unix&from=nlp_qa&id=312754&q=how%20to%20strip%20metadata%20from%20image%20files&s=unix.stackexchange.com&tl=How%20to%20strip%20metadata%20from%20image%20files%20%2D%20Unix%20%26%20Linux%20Stack%20Exchange + preg_match( + '/nrj\(\'(\/a\.js\?.*from=nlp_qa.*)\'\)/U', + $js, + $stack + ); + + if(isset($stack[1])){ + + $stack = $stack[1]; + + try{ + $stackjs = $this->get( + "https://duckduckgo.com" . $stack, + [], + ddg::req_xhr + ); + + if( + !preg_match( + '/^DDG\.duckbar\.failed/', + $stackjs + ) + ){ + + preg_match( + '/DDG\.duckbar\.add_array\((\[\{[\S\s]*}])\)/U', + $stackjs, + $stackjson + ); + + $stackjson = json_decode($stackjson[1], true)[0]["data"][0]; + + $out["answer"][] = [ + "title" => $stackjson["Heading"], + "description" => $this->htmltoarray($stackjson["Abstract"]), + "url" => str_replace(["http://", "ddg"], ["https://", ""], $stackjson["AbstractURL"]), + "thumb" => null, + "table" => [], + "sublink" => [] + ]; + } + + }catch(Exception $e){ + + // do nothing + } + } + + /* + Check for musicmatch (lyrics) + */ + preg_match( + '/nrj\(\'(\/a\.js\?.*&s=lyrics.*)\'\)/U', + $js, + $lyrics + ); + + if(isset($lyrics[1])){ + + $lyrics = $lyrics[1]; + + try{ + $lyricsjs = $this->get( + "https://duckduckgo.com" . $lyrics, + [], + ddg::req_xhr + ); + + if( + !preg_match( + '/^DDG\.duckbar\.failed/', + $lyricsjs + ) + ){ + + preg_match( + '/DDG\.duckbar\.add_array\((\[\{[\S\s]*}])\)/U', + $lyricsjs, + $lyricsjson + ); + + $lyricsjson = json_decode($lyricsjson[1], true)[0]["data"][0]; + + $title = null; + + if(isset($lyricsjson["Heading"])){ + + $title = $lyricsjson["Heading"]; + }elseif(isset($lyricsjson["data"][1]["urlTitle"])){ + + $title = $lyricsjson["data"][1]["urlTitle"]; + }else{ + + $title = $lyricsjson["data"][0]["song_title"]; + } + + $description = [ + [ + "type" => "text", + "value" => null + ] + ]; + $parts = + explode( + "<br>", + str_ireplace( + ["<br>", "</br>", "<br/>"], + "<br>", + $lyricsjson["Abstract"] + ), + ); + + for($i=0; $i<count($parts); $i++){ + + $description[0]["value"] .= trim($parts[$i]) . "\n"; + } + + $description[0]["value"] = trim($description[0]["value"]); + + $description[] = + [ + "type" => "quote", + "value" => + "Written by " . implode(", ", $lyricsjson["data"][0]["writers"]) . + "\nFrom the album " . $lyricsjson["data"][0]["albums"][0]["title"] . + "\nReleased on the " . date("jS \of F Y", strtotime($lyricsjson["data"][0]["albums"][0]["release_date"])) + ]; + + $out["answer"][] = [ + "title" => $title, + "description" => $description, + "url" => $lyricsjson["AbstractURL"], + "thumb" => null, + "table" => [], + "sublink" => [] + ]; + } + + }catch(Exception $e){ + + // do nothing + } + } + } + + /* + Get related searches + */ + preg_match( + '/DDG\.duckbar\.loadModule\(\'related_searches\', ?{[\s\S]*"results":(\[{[\s\S]*}]),"vqd"/U', + $js, + $related + ); + + if(isset($related[1])){ + + try{ + $related = json_decode($related[1], true); + + for($i=0; $i<count($related); $i++){ + + if(isset($related[$i]["text"])){ + + array_push($out["related"], $related[$i]["text"]); + } + } + + }catch(Exception $e){ + + // do nothing + } + } + + unset($related); + + /* + Get answers + */ + $answer_count = preg_match_all( + '/DDG\.duckbar\.add\(({.*[\S\s]*})(?:\);|,null,"index"\))/U', + $js . $inithtml, + $answers + ); + + try{ + + if(isset($answers[1])){ + + $answers = $answers[1]; + + for($i=0; $i<$answer_count; $i++){ + + $answers[$i] = json_decode($answers[$i], true); + + // remove dupes + for($k=0; $k<count($out["answer"]); $k++){ + + if( + !isset($answers[$i]["data"]["AbstractURL"]) || + str_replace("_", "%20", $out["answer"][$k]["url"]) == str_replace("_", "%20", $this->sanitizeurl($answers[$i]["data"]["AbstractURL"])) + ){ + + continue 2; + } + } + + // get more related queries + if( + isset($answers[$i]["data"]["RelatedTopics"]) && + $answers[$i]["data"]["RelatedTopics"] != 0 + ){ + + for($k=0; $k<count($answers[$i]["data"]["RelatedTopics"]); $k++){ + + if(isset($answers[$i]["data"]["RelatedTopics"][$k]["Result"])){ + + preg_match( + '/">(.*)<\//', + $answers[$i]["data"]["RelatedTopics"][$k]["Result"], + $label + ); + + array_push($out["related"], htmlspecialchars_decode(strip_tags($label[1]))); + } + } + } + + $image = null; + + // get image + if( + isset($answers[$i]["data"]["Image"]) && + !empty($answers[$i]["data"]["Image"]) && + $answers[$i]["data"]["Image"] != "https://duckduckgo.com/i/" + ){ + if(strpos($answers[$i]["data"]["Image"], "https://duckduckgo.com/i/") === true){ + + $image = $answers[$i]["data"]["Image"]; + }else{ + + if( + strlen($answers[$i]["data"]["Image"]) > 0 && + $answers[$i]["data"]["Image"][0] == "/" + ){ + + $answers[$i]["data"]["Image"] = substr($answers[$i]["data"]["Image"], 1); + } + + $image = "https://duckduckgo.com/" . $answers[$i]["data"]["Image"]; + } + } + + $count = count($out["answer"]); + + if(isset($answers[$i]["data"]["AbstractText"]) && !empty($answers[$i]["data"]["AbstractText"])){ + + $description = $this->htmltoarray($answers[$i]["data"]["AbstractText"]); + }elseif(isset($answers[$i]["data"]["Abstract"]) && !empty($answers[$i]["data"]["Abstract"])){ + + $description = $this->htmltoarray($answers[$i]["data"]["Abstract"]); + }elseif(isset($answers[$i]["data"]["Answer"]) && !empty($answers[$i]["data"]["Answer"])){ + + $description = $this->htmltoarray($answers[$i]["data"]["Answer"]); + }else{ + + $description = []; + } + + if(isset($answers[$i]["data"]["Heading"]) && !empty($answers[$i]["data"]["Heading"])){ + + $title = $this->unescapehtml($answers[$i]["data"]["Heading"]); + }else{ + + // no title, ignore bs + continue; + //$title = null; + } + + if(isset($answers[$i]["data"]["AbstractURL"]) && !empty($answers[$i]["data"]["AbstractURL"])){ + + $url = $answers[$i]["data"]["AbstractURL"]; + }else{ + + $url = null; + } + + $out["answer"][$count] = [ + "title" => $title, + "description" => $description, + "url" => $this->sanitizeurl($url), + "thumb" => $image, + "table" => [], + "sublink" => [] + ]; + + if(isset($answers[$i]["data"]["Infobox"]["content"])){ + + for($k=0; $k<count($answers[$i]["data"]["Infobox"]["content"]); $k++){ + + // populate table + if($answers[$i]["data"]["Infobox"]["content"][$k]["data_type"] == "string"){ + + $out["answer"][$count]["table"][$answers[$i]["data"]["Infobox"]["content"][$k]["label"]] = + $answers[$i]["data"]["Infobox"]["content"][$k]["value"]; + continue; + } + + $url = ""; + $type = "Website"; + + switch($answers[$i]["data"]["Infobox"]["content"][$k]["data_type"]){ + case "official_site": + case "official_website": + $type = "Website"; + break; + + case "wikipedia": $type = "Wikipedia"; break; + case "itunes": $type = "iTunes"; break; + case "amazon": $type = "Amazon"; break; + + case "imdb_title_id": + case "imdb_id": + case "imdb_name_id": + $type = "IMDb"; + $delim = substr($answers[$i]["data"]["Infobox"]["content"][$k]["value"], 0, 2); + + if($delim == "nm"){ + + $url = "https://www.imdb.com/name/"; + }elseif($delim == "tt"){ + + $url = "https://www.imdb.com/title/"; + }elseif($delim == "co"){ + + $url = "https://www.imdb.com/search/title/?companies="; + }else{ + + $url = "https://www.imdb.com/title/"; + } + break; + + case "imdb_name_id": $url = "https://www.imdb.com/name/"; $type = "IMDb"; break; + case "twitter_profile": $url = "https://twitter.com/"; $type = "Twitter"; break; + case "instagram_profile": $url = "https://instagram.com/"; $type = "Instagram"; break; + case "facebook_profile": $url = "https://facebook.com/"; $type = "Facebook"; break; + case "spotify_artist_id": $url = "https://open.spotify.com/artist/"; $type = "Spotify"; break; + case "rotten_tomatoes": $url = "https://rottentomatoes.com/"; $type = "Rotten Tomatoes"; break; + case "youtube_channel": $url = "https://youtube.com/channel/"; $type = "YouTube"; break; + case "soundcloud_id": $url = "https://soundcloud.com/"; $type = "SoundCloud"; break; + + default: + continue 2; + } + + // populate sublinks + $out["answer"][$count]["sublink"][$type] = + $url . $answers[$i]["data"]["Infobox"]["content"][$k]["value"]; + } + } + } + } + + }catch(Exception $e){ + + // do nothing + } + + /* + Get shitcoin conversions + */ + if($extendedsearch){ + if( + preg_match( + '/"https?:\/\/(?:www\.coinbase\.com\/converter\/([a-z0-9]+)\/([a-z0-9]+)|changelly\.com\/exchange\/([a-z0-9]+)\/([a-z0-9]+)|coinmarketcap\.com\/currencies\/[a-z0-9]+\/([a-z0-9]+)\/([a-z0-9]+))\/?"/', + $js, + $shitcoins + ) + ){ + + $shitcoins = array_values(array_filter($shitcoins)); + + preg_match( + '/(?:[\s,.]*[0-9]+)+/', + $search, + $amount + ); + + if(count($amount) === 1){ + + $amount = (float)str_replace([" ", ","], ["", "."], $amount[0]); + }else{ + + $amount = 1; + } + + try{ + + $description = []; + + $shitcoinjs = $this->get( + "https://duckduckgo.com/js/spice/cryptocurrency/{$shitcoins[1]}/{$shitcoins[2]}/1", + [], + ddg::req_xhr + ); + + preg_match( + '/ddg_spice_cryptocurrency\(\s*({[\S\s]*})\s*\);/', + $shitcoinjs, + $shitcoinjson + ); + + $shitcoinjson = json_decode($shitcoinjson[1], true); + + if( + !isset($shitcoinjson["error"]) && + $shitcoinjson["status"]["error_code"] == 0 + ){ + + $shitcoinjson = $shitcoinjson["data"]; + $array_values = array_values($shitcoinjson["quote"])[0]; + + if($amount != 1){ + + // show conversion + $description[] = [ + "type" => "title", + "value" => "Conversion" + ]; + + $description[] = [ + "type" => "text", + "value" => + "{$amount} {$shitcoinjson["name"]} ({$shitcoinjson["symbol"]}) = " . $this->number_format($array_values["price"] * $amount) . " " . strtoupper($shitcoins[2]) . "\n" . + "{$amount} " . strtoupper($shitcoins[2]) . " = " . $this->number_format((1 / $array_values["price"]) * $amount) . " {$shitcoinjson["symbol"]}" + ]; + } + + $description[] = [ + "type" => "title", + "value" => "Current rates" + ]; + + // rates + $description[] = [ + "type" => "text", + "value" => + "1 {$shitcoinjson["name"]} ({$shitcoinjson["symbol"]}) = " . $this->number_format($array_values["price"]) . " " . strtoupper($shitcoins[2]) . "\n" . + "1 " . strtoupper($shitcoins[2]) . " = " . $this->number_format(1 / $array_values["price"]) . " {$shitcoinjson["symbol"]}" + ]; + + $description[] = [ + "type" => "quote", + "value" => "Last fetched: " . date("jS \of F Y @ g:ia", strtotime($shitcoinjson["last_updated"])) + ]; + + $out["answer"][] = [ + "title" => $shitcoinjson["name"] . " (" . strtoupper($shitcoins[1]) . ") & " . strtoupper($shitcoins[2]) . " market", + "description" => $description, + "url" => "https://coinmarketcap.com/converter/" . strtoupper($shitcoins[1]) . "/" . strtoupper($shitcoins[2]) . "/?amt={$amount}", + "thumb" => null, + "table" => [], + "sublink" => [] + ]; + } + + }catch(Exception $e){ + + // do nothing + } + }else{ + + /* + Get currency conversion + */ + if( + preg_match( + '/"https:\/\/www\.xe\.com\/currencyconverter\/convert\/\?From=([A-Z0-9]+)&To=([A-Z0-9]+)"/', + $js, + $currencies + ) + ){ + + preg_match( + '/(?:[\s,.]*[0-9]+)+/', + $search, + $amount + ); + + if(count($amount) === 1){ + + $amount = (float)str_replace([" ", ","], ["", "."], $amount[0]); + }else{ + + $amount = 1; + } + + try{ + $currencyjs = $this->get( + "https://duckduckgo.com/js/spice/currency/{$amount}/" . strtolower($currencies[1]) . "/" . strtolower($currencies[2]), + [], + ddg::req_xhr + ); + + preg_match( + '/ddg_spice_currency\(\s*({[\S\s]*})\s*\);/', + $currencyjs, + $currencyjson + ); + + $currencyjson = json_decode($currencyjson[1], true); + + if(empty($currencyjson["headers"]["description"])){ + + $currencyjson = $currencyjson["conversion"]; + $description = []; + + if($amount != 1){ + + $description[] = + [ + "type" => "title", + "value" => "Conversion" + ]; + + $description[] = + [ + "type" => "text", + "value" => + $this->number_format($currencyjson["from-amount"]) . " {$currencyjson["from-currency-symbol"]} = " . + $this->number_format($currencyjson["converted-amount"]) . " {$currencyjson["to-currency-symbol"]}" + ]; + } + + $description[] = + [ + "type" => "title", + "value" => "Current rates" + ]; + + $description[] = + [ + "type" => "text", + "value" => + "{$currencyjson["conversion-rate"]}\n" . + "{$currencyjson["conversion-inverse"]}" + ]; + + $description[] = + [ + "type" => "quote", + "value" => "Last fetched: " . date("jS \of F Y @ g:ia", strtotime($currencyjson["rate-utc-timestamp"])) + ]; + + $out["answer"][] = [ + "title" => + "{$currencyjson["from-currency-name"]} ({$currencyjson["from-currency-symbol"]}) to " . + "{$currencyjson["to-currency-name"]} ({$currencyjson["to-currency-symbol"]})", + "description" => $description, + "url" => "https://www.xe.com/currencyconverter/convert/?Amount={$amount}&From={$currencies[1]}&To={$currencies[2]}", + "thumb" => null, + "table" => [], + "sublink" => [] + ]; + } + + }catch(Exception $e){ + + // do nothing + } + } + } + } + + /* + Get small answer + */ + preg_match( + '/DDG\.ready\(function ?\(\) ?{DDH\.add\(({[\S\s]+}),"index"\)}\)/U', + $inithtml, + $smallanswer + ); + + if(isset($smallanswer[1])){ + + $smallanswer = json_decode($smallanswer[1], true); + + if( + !isset($smallanswer["require"]) && + isset($smallanswer["data"]["title"]) + ){ + + if(isset($smallanswer["data"]["url"])){ + + $url = $this->unescapehtml($smallanswer["data"]["url"]); + }elseif(isset($smallanswer["meta"]["sourceUrl"])){ + + $url = $this->unescapehtml($smallanswer["meta"]["sourceUrl"]); + }else{ + + $url = null; + } + + $out["answer"] = [ + [ + "title" => $this->unescapehtml($smallanswer["data"]["title"]), + "description" => [], + "url" => $this->sanitizeurl($url), + "thumb" => null, + "table" => [], + "sublink" => [] + ], + ...$out["answer"] + ]; + + if(isset($smallanswer["data"]["subtitle"])){ + + $out["answer"][0]["description"][] = + [ + "type" => "text", + "value" => isset($smallanswer["data"]["subtitle"]) ? $this->unescapehtml($smallanswer["data"]["subtitle"]) : null + ]; + } + } + } + + unset($inithtml); + unset($answers); + unset($answer_count); + + /* + Get spelling autocorrect + */ + + preg_match( + '/DDG\.page\.showMessage\(\'spelling\',({[\S\s]+})\)/U', + $js, + $spelling + ); + + if(isset($spelling[1])){ + + $spelling = json_decode($spelling[1], true); + + switch((int)$spelling["qc"]){ + + case 1: + case 3: + case 5: + $type = "including"; + break; + + default: + $type = "not_many"; + break; + } + + $out["spelling"] = [ + "type" => $type, + "using" => $this->unescapehtml(strip_tags($spelling["suggestion"])), + "correction" => $this->unescapehtml(strip_tags($spelling["recourseText"])) + ]; + } + + unset($spelling); + + /* + Get web results + */ + preg_match( + '/DDG\.pageLayout\.load\(\'d\', ?(\[{"[\S\s]*"}])\)/U', + $js, + $web + ); + + if(isset($web[1])){ + + try{ + $web = json_decode($web[1], true); + + for($i=0; $i<count($web); $i++){ + + // ignore google placeholder + fake next page + if( + isset($web[$i]["t"]) && + ( + $web[$i]["t"] == "EOP" || + $web[$i]["t"] == "EOF" + ) && + strpos($web[$i]["c"], "://www.google.") !== false + ){ + + break; + } + + // store next page token + if(isset($web[$i]["n"])){ + + $out["npt"] = $this->nextpage->store($web[$i]["n"] . "&biaexp=b&eslexp=a&litexp=c&msvrtexp=b&wrap=1", "web"); + continue; + } + + // ignore malformed data + if(!isset($web[$i]["t"])){ + + continue; + } + + $sublinks = []; + + if(isset($web[$i]["l"])){ + + for($k=0; $k<count($web[$i]["l"]); $k++){ + + if( + !isset($web[$i]["l"][$k]["targetUrl"]) || + !isset($web[$i]["l"][$k]["text"]) + ){ + + continue; + } + + array_push( + $sublinks, + [ + "title" => $this->titledots($this->unescapehtml($web[$i]["l"][$k]["text"])), + "date" => null, + "description" => isset($web[$i]["l"][$k]["snippet"]) ? $this->titledots($this->unescapehtml($web[$i]["l"][$k]["snippet"])) : null, + "url" => $this->sanitizeurl($web[$i]["l"][$k]["targetUrl"]) + ] + ); + } + } + + if( + preg_match( + '/^<span class="result__type">PDF<\/span>/', + $web[$i]["t"] + ) + ){ + + $type = "pdf"; + $web[$i]["t"] = + str_replace( + '<span class="result__type">PDF</span>', + "", + $web[$i]["t"] + ); + }else{ + + $type = "web"; + } + + if(isset($web[$i]["e"])){ + + $date = strtotime($web[$i]["e"]); + }else{ + + $date = null; + } + + array_push( + $out["web"], + [ + "title" => $this->titledots($this->unescapehtml(strip_tags($web[$i]["t"]))), + "description" => $this->titledots($this->unescapehtml(strip_tags($web[$i]["a"]))), + "url" => isset($web[$i]["u"]) ? $this->sanitizeurl($web[$i]["u"]) : $this->sanitizeurl($web[$i]["c"]), + "date" => $date, + "type" => $type, + "thumb" => + [ + "url" => null, + "ratio" => null + ], + "sublink" => $sublinks, + "table" => [] + ] + ); + } + + }catch(Exception $e){ + + // do nothing + } + } + + unset($web); + + /* + Get images + */ + preg_match( + '/DDG\.duckbar\.load\(\'images\', ?{[\s\S]*"results":(\[{"[\s\S]*}]),"vqd"/U', + $js, + $images + ); + + if(isset($images[1])){ + + try{ + $images = json_decode($images[1], true); + + for($i=0; $i<count($images); $i++){ + + if( + !isset($images[$i]["title"]) || + !isset($images[$i]["image"]) || + !isset($images[$i]["thumbnail"]) || + !isset($images[$i]["width"]) || + !isset($images[$i]["height"]) + ){ + + continue; + } + + $ratio = + $this->bingratio( + (int)$images[$i]["width"], + (int)$images[$i]["height"] + ); + + array_push( + $out["image"], + [ + "title" => $this->titledots($this->unescapehtml($images[$i]["title"])), + "source" => [ + [ + "url" => $images[$i]["image"], + "width" => (int)$images[$i]["width"], + "height" => (int)$images[$i]["height"] + ], + [ + "url" => $this->bingimg($images[$i]["thumbnail"]), + "width" => $ratio[0], + "height" => $ratio[1] + ] + ], + "url" => $this->sanitizeurl($images[$i]["url"]) + ] + ); + } + + }catch(Exception $e){ + + // do nothing + } + } + + unset($images); + + /* + Get videos + */ + preg_match( + '/DDG\.duckbar\.load\(\'videos\', ?{[\s\S]*"results":(\[{"[\s\S]*}]),"vqd"/U', + $js, + $videos + ); + + if(isset($videos[1])){ + try{ + $videos = json_decode($videos[1], true); + + for($i=0; $i<count($videos); $i++){ + + $cachekey = false; + + foreach(["large", "medium", "small"] as &$key){ + + if(isset($videos[$i]["images"][$key])){ + + $cachekey = $key; + break; + } + } + + if( + !isset($videos[$i]["title"]) || + !isset($videos[$i]["description"]) || + $cachekey === false || + !isset($videos[$i]["content"]) + ){ + + continue; + } + + array_push( + $out["video"], + [ + "title" => $this->titledots($this->unescapehtml($videos[$i]["title"])), + "description" => $videos[$i]["description"] == "" ? null : $this->titledots($this->unescapehtml($videos[$i]["description"])), + "date" => $videos[$i]["published"] == "" ? null : strtotime($videos[$i]["published"]), + "duration" => $videos[$i]["duration"] == 0 ? null : $this->hmstoseconds($videos[$i]["duration"]), + "views" => $videos[$i]["statistics"]["viewCount"] == 0 ? null : $videos[$i]["statistics"]["viewCount"], + "thumb" => + [ + "url" => $this->bingimg($videos[$i]["images"][$cachekey]), + "ratio" => "16:9" + ], + "url" => $this->sanitizeurl($videos[$i]["content"]) + ] + ); + } + + }catch(Exception $e){ + + // do nothing + } + } + + unset($videos); + + /* + Get news + */ + preg_match( + '/DDG\.duckbar\.load\(\'news\', ?{[\s\S]*"results":(\[{"[\s\S]*}]),"vqd"/U', + $js, + $news + ); + + if(isset($news[1])){ + try{ + $news = json_decode($news[1], true); + + for($i=0; $i<count($news); $i++){ + + if( + !isset($news[$i]["title"]) || + !isset($news[$i]["excerpt"]) || + !isset($news[$i]["url"]) + ){ + + continue; + } + + array_push( + $out["news"], + [ + "title" => $this->titledots($this->unescapehtml($news[$i]["title"])), + "description" => $this->titledots($this->unescapehtml(strip_tags($news[$i]["excerpt"]))), + "date" => isset($news[$i]["date"]) ? (int)$news[$i]["date"] : null, + "thumb" => + [ + "url" => isset($news[$i]["image"]) ? $news[$i]["image"] : null, + "ratio" => "16:9" + ], + "url" => $this->sanitizeurl($news[$i]["url"]) + ] + ); + } + + }catch(Exception $e){ + + // do nothing + } + } + + return $out; + } + + public function image($get){ + + if($get["npt"]){ + + $npt = $this->nextpage->get($get["npt"], "images"); + + try{ + $json = json_decode($this->get( + "https://duckduckgo.com/i.js?" . $npt, + [], + ddg::req_xhr + ), true); + + }catch(Exception $err){ + + throw new Exception("Failed to get i.js"); + } + + }else{ + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $country = $get["country"]; + $nsfw = $get["nsfw"]; + $date = $get["date"]; + $size = $get["size"]; + $color = $get["color"]; + $type = $get["type"]; + $layout = $get["layout"]; + $license = $get["license"]; + + $filter = []; + $get_filters = [ + "q" => $search, + "iax" => "images", + "ia" => "images" + ]; + + if($date != "any"){ $filter[] = "time:$date"; } + if($size != "any"){ $filter[] = "size:$size"; } + if($color != "any"){ $filter[] = "color:$color"; } + if($type != "any"){ $filter[] = "type:$type"; } + if($layout != "any"){ $filter[] = "layout:$layout"; } + if($license != "any"){ $filter[] = "license:$license"; } + + $filter = implode(",", $filter); + + if($filter != ""){ + + $get_filters["iaf"] = $filter; + } + + switch($nsfw){ + + case "yes": $get_filters["kp"] = "-2"; break; + case "no": $get_filters["kp"] = "-1"; break; + } + + try{ + + $html = $this->get( + "https://duckduckgo.com", + $get_filters, + ddg::req_web + ); + }catch(Exception $err){ + + throw new Exception("Failed to get html"); + } + + preg_match( + '/vqd=([0-9-]+)/', + $html, + $vqd + ); + + if(!isset($vqd[1])){ + + throw new Exception("Failed to get vqd token"); + } + + $vqd = $vqd[1]; + + // @TODO: s param = image offset + $js_params = [ + "l" => $country, + "o" => "json", + "q" => $search, + "vqd" => $vqd + ]; + + switch($nsfw){ + + case "yes": $js_params["p"] = "-1"; break; + case "no": $js_params["p"] = "1"; break; + } + + if(empty($filter)){ + + $js_params["f"] = "1"; + }else{ + + $js_params["f"] = $filter; + } + + try{ + $json = json_decode($this->get( + "https://duckduckgo.com/i.js", + $js_params, + ddg::req_xhr + ), true); + + }catch(Exception $err){ + + throw new Exception("Failed to get i.js"); + } + } + + $out = [ + "status" => "ok", + "npt" => null, + "image" => [] + ]; + + if(isset($json["next"])){ + + if(!isset($vqd)){ + + $vqd = array_values($json["vqd"])[0]; + } + + $out["npt"] = + $this->nextpage->store( + explode("?", $json["next"])[1] . "&vqd=" . + $vqd, + "images" + ); + } + + for($i=0; $i<count($json["results"]); $i++){ + + $bingimg = $this->bingimg($json["results"][$i]["thumbnail"]); + $ratio = + $this->bingratio( + (int)$json["results"][$i]["width"], + (int)$json["results"][$i]["height"] + ); + + $out["image"][] = [ + "title" => $this->titledots($this->unescapehtml($json["results"][$i]["title"])), + "source" => [ + [ + "url" => $json["results"][$i]["image"], + "width" => (int)$json["results"][$i]["width"], + "height" => (int)$json["results"][$i]["height"] + ], + [ + "url" => $bingimg, + "width" => $ratio[0], + "height" => $ratio[1], + ] + ], + "url" => $this->sanitizeurl($json["results"][$i]["url"]) + ]; + } + + return $out; + } + + public function video($get){ + + if($get["npt"]){ + + $npt = $this->nextpage->get($get["npt"], "videos"); + + try{ + $json = json_decode($this->get( + "https://duckduckgo.com/v.js?" . + $npt, + [], + ddg::req_xhr + ), true); + + }catch(Exception $err){ + + throw new Exception("Failed to get v.js"); + } + }else{ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $country = $get["country"]; + $nsfw = $get["nsfw"]; + $date = $get["date"]; + $resolution = $get["resolution"]; + $duration = $get["duration"]; + $license = $get["license"]; + + $filter = []; + + $get_filters = [ + "q" => $search, + "iax" => "videos", + "ia" => "videos" + ]; + + switch($nsfw){ + + case "yes": $get_filters["kp"] = "-2"; break; + case "no": $get_filters["kp"] = "-1"; break; + } + + if($date != "any"){ $filter[] = "publishedAfter:{$date}"; } + if($resolution != "any"){ $filter[] = "videoDefinition:{$resolution}"; } + if($duration != "any"){ $filter[] = "videoDuration:{$duration}"; } + if($license != "any"){ $filter[] = "videoLicense:{$license}"; } + + $filter = implode(",", $filter); + + try{ + + $html = $this->get( + "https://duckduckgo.com", + $get_filters, + ddg::req_web + ); + }catch(Exception $err){ + + throw new Exception("Failed to get html"); + } + + preg_match( + '/vqd=([0-9-]+)/', + $html, + $vqd + ); + + if(!isset($vqd[1])){ + + throw new Exception("Failed to get vqd token"); + } + + $vqd = $vqd[1]; + + try{ + $json = json_decode($this->get( + "https://duckduckgo.com/v.js", + [ + "l" => "us-en", + "o" => "json", + "sr" => 1, + "q" => $search, + "vqd" => $vqd, + "f" => $filter, + "p" => $get_filters["kp"] + ], + ddg::req_xhr + ), true); + + }catch(Exception $err){ + + throw new Exception("Failed to get v.js"); + } + } + + $out = [ + "status" => "ok", + "npt" => null, + "video" => [], + "author" => [], + "livestream" => [], + "playlist" => [], + "reel" => [] + ]; + + if(isset($json["next"])){ + + $out["npt"] = + $this->nextpage->store( + explode("?", $json["next"])[1], + "videos" + ); + } + + for($i=0; $i<count($json["results"]); $i++){ + + $cachekey = false; + + foreach(["large", "medium", "small"] as &$key){ + + if(isset($json["results"][$i]["images"][$key])){ + + $cachekey = $key; + break; + } + } + + if( + !isset($json["results"][$i]["title"]) || + !isset($json["results"][$i]["description"]) || + $cachekey === false || + !isset($json["results"][$i]["content"]) + ){ + + continue; + } + + array_push( + $out["video"], + [ + "title" => $this->titledots($this->unescapehtml($json["results"][$i]["title"])), + "description" => $json["results"][$i]["description"] == "" ? null : $this->titledots($this->unescapehtml($json["results"][$i]["description"])), + "author" => [ + "name" => empty($json["results"][$i]["uploader"]) ? null : $this->unescapehtml($json["results"][$i]["uploader"]), + "url" => null, + "avatar" => null + ], + "date" => $json["results"][$i]["published"] == "" ? null : strtotime($json["results"][$i]["published"]), + "duration" => $json["results"][$i]["duration"] == 0 ? null : $this->hmstoseconds($json["results"][$i]["duration"]), + "views" => $json["results"][$i]["statistics"]["viewCount"] == 0 ? null : $json["results"][$i]["statistics"]["viewCount"], + "thumb" => [ + "url" => $this->bingimg($json["results"][$i]["images"][$cachekey]), + "ratio" => "16:9" + ], + "url" => $this->sanitizeurl($json["results"][$i]["content"]) + ] + ); + } + + return $out; + } + + public function news($get){ + + if($get["npt"]){ + + $req = $this->nextpage->get($get["npt"], "news"); + + try{ + + $json = json_decode($this->get( + "https://duckduckgo.com/news.js?" . + $req, + [], + ddg::req_xhr + ), true); + + }catch(Exception $err){ + + throw new Exception("Failed to get news.js"); + } + }else{ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $country = $get["country"]; + $nsfw = $get["nsfw"]; + $date = $get["date"]; + + $get_params = [ + "q" => $search, + "iar" => "news", + "ia" => "news" + ]; + + switch($nsfw){ + + case "yes": $get_filters["kp"] = "-2"; break; + case "maybe": $get_filters["kp"] = "-1"; break; + case "no": $get_filters["kp"] = "1"; break; + } + + if($date != "any"){ + + $get_params["df"] = $date; + } + + try{ + + $html = $this->get( + "https://duckduckgo.com", + $get_params, + ddg::req_web + ); + }catch(Exception $err){ + + throw new Exception("Failed to get html"); + } + + preg_match( + '/vqd=([0-9-]+)/', + $html, + $vqd + ); + + if(!isset($vqd[1])){ + + throw new Exception("Failed to get vqd token"); + } + + $vqd = $vqd[1]; + + try{ + + $js_params = [ + "l" => $country, + "o" => "json", + "noamp" => "1", + "q" => $search, + "vqd" => $vqd, + "p" => $get_filters["kp"] + ]; + + if($date != "any"){ + + $js_params["df"] = $date; + }else{ + + $js_params["df"] = ""; + } + + $json = json_decode($this->get( + "https://duckduckgo.com/news.js", + $js_params, + ddg::req_xhr + ), true); + + }catch(Exception $err){ + + throw new Exception("Failed to get news.js"); + } + } + + $out = [ + "status" => "ok", + "npt" => null, + "news" => [] + ]; + + if(isset($json["next"])){ + + $out["npt"] = + $this->nextpage->store( + explode("?", $json["next"])[1], + "news" + ); + } + + for($i=0; $i<count($json["results"]); $i++){ + + $out["news"][] = [ + "title" => $this->titledots($this->unescapehtml($json["results"][$i]["title"])), + "author" => $this->unescapehtml($json["results"][$i]["source"]), + "description" => $this->titledots($this->unescapehtml(strip_tags($json["results"][$i]["excerpt"]))), + "date" => $json["results"][$i]["date"], + "thumb" => + [ + "url" => isset($json["results"][$i]["image"]) ? $json["results"][$i]["image"] : null, + "ratio" => "16:9" + ], + "url" => $this->sanitizeurl($json["results"][$i]["url"]) + ]; + } + + return $out; + } + + private function hmstoseconds($time){ + + $parts = explode(":", $time, 3); + $time = 0; + + if(count($parts) === 3){ + + // hours + $time = $time + ((int)$parts[0] * 3600); + array_shift($parts); + } + + if(count($parts) === 2){ + + // minutes + $time = $time + ((int)$parts[0] * 60); + array_shift($parts); + } + + // seconds + $time = $time + (int)$parts[0]; + + return $time; + } + + private function titledots($title){ + + $substr = substr($title, -3); + + if( + $substr == "..." || + $substr == "…" + ){ + + return trim(substr($title, 0, -3)); + } + + return trim($title); + } + + private function unescapehtml($str){ + + return html_entity_decode( + str_replace( + [ + "<br>", + "<br/>", + "</br>", + "<BR>", + "<BR/>", + "</BR>", + ], + "\n", + $str + ), + ENT_QUOTES | ENT_XML1, 'UTF-8' + ); + } + + private function bingimg($url){ + + $parse = parse_url($url); + parse_str($parse["query"], $parts); + + return "https://" . $parse["host"] . "/th?id=" . urlencode($parts["id"]); + } + + private function htmltoarray($html){ + + $html = strip_tags($html, ["img", "pre", "code", "br", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote", "a"]); + + libxml_use_internal_errors(true); + $dom = new DOMDocument("1.0", "utf-8"); + $dom->loadHTML('<div>' . $html . '</div>'); + $xpath = new DOMXPath($dom); + $descendants = $xpath->query('//div/node()'); + + $images = $xpath->query('//div/node()/img'); + $imageiterator = 0; + + if(count($descendants) === 0){ + + return [ + "type" => "text", + "value" => $this->unescapehtml($html) + ]; + } + + $array = []; + $previoustype = null; + + foreach($descendants as $node){ + + // $node->nodeValue = iconv("UTF-8", "ISO-8859-1//TRANSLIT", $node->nodeValue); + + // get node type + switch($node->nodeName){ + case "#text": + $type = "text"; + break; + + case "pre": + $type = "code"; + break; + + case "code": + $type = "inline_code"; + break; + + case "h1": + case "h2": + case "h3": + case "h4": + case "h5": + case "h6": + $type = "title"; + break; + + case "blockquote": + $type = "quote"; + break; + + case "a": + $type = "link"; + break; + + case "img": + $type = "image"; + break; + } + + // add node to array + switch($type){ + + case "text": + $value = preg_replace( + '/ {2,}/', + " ", + $this->limitnewlines($this->unescapehtml($node->textContent)) + ); + + if( + $previoustype == "quote" || + $previoustype === null || + $previoustype == "image" || + $previoustype == "title" || + $previoustype == "code" + ){ + + $value = ltrim($value); + } + + if($value == ""){ + + $previoustype = $type; + continue 2; + } + + // merge with previous text node + if($previoustype == "text"){ + + $array[count($array) - 1]["value"] = trim($array[count($array) - 1]["value"]) . "\n" . $this->bstoutf8($value); + }else{ + + $array[] = [ + "type" => "text", + "value" => $this->bstoutf8($value) + ]; + } + break; + + case "inline_code": + case "bold": + $array[] = [ + "type" => "inline_code", + "value" => $this->bstoutf8(trim($this->limitnewlines($this->unescapehtml($node->textContent)))) + ]; + break; + + case "link": + // check for link nested inside of image + + if(strlen($node->childNodes->item(0)->textContent) !== 0){ + + $array[] = [ + "type" => "link", + "value" => $this->bstoutf8(trim($this->unescapehtml($node->textContent))), + "url" => $this->bstoutf8(preg_replace('/\/ddg$/', "", preg_replace('/^http:\/\//', "https://", $this->sanitizeurl($node->getAttribute("href"))))) + ]; + break; + } + + $type = "image"; + + if($previoustype == "text"){ + + $array[count($array) - 1]["value"] = rtrim($array[count($array) - 1]["value"]); + } + + $array[] = [ + "type" => "image", + "url" => $this->bstoutf8(preg_replace('/^http:\/\//', "https://", preg_replace('/^\/\/images\.duckduckgo\.com\/iu\/\?u=/', "", $images->item($imageiterator)->getAttribute("src")))) + ]; + + $imageiterator++; + + break; + + case "image": + + if($previoustype == "text"){ + + $array[count($array) - 1]["value"] = rtrim($array[count($array) - 1]["value"]); + } + + $array[] = [ + "type" => "image", + "url" => $this->bstoutf8(preg_replace('/^http:\/\//', "https://", preg_replace('/^\/\/images\.duckduckgo\.com\/iu\/\?u=/', "", $node->getAttribute("src")))) + ]; + break; + + case "quote": + case "title": + case "code": + if($previoustype == "text"){ + + $array[count($array) - 1]["value"] = rtrim($array[count($array) - 1]["value"]); + } + // no break + + default: + + $value = trim($this->limitnewlines($this->unescapehtml($node->textContent))); + if($type != "code"){ + + $value = preg_replace( + '/ {2,}/', + " ", + $value + ); + } + + $array[] = [ + "type" => $type, + "value" => $this->bstoutf8($value) + ]; + break; + } + + $previoustype = $type; + } + + return $array; + } + + private function bstoutf8($bs){ + + return iconv("UTF-8", "ISO-8859-1//TRANSLIT", $bs); + } + + private function limitnewlines($text){ + + preg_replace( + '/(?:[\n\r] *){2,}/m', + "\n\n", + $text + ); + + return $text; + } + + private function sanitizeurl($url){ + + // check for domains w/out first short subdomain (ex: www.) + + $domain = parse_url($url, PHP_URL_HOST); + + $subdomain = preg_replace( + '/^[A-z0-9]{1,3}\./', + "", + $domain + ); + + switch($subdomain){ + case "ebay.com.au": + case "ebay.at": + case "ebay.ca": + case "ebay.fr": + case "ebay.de": + case "ebay.com.hk": + case "ebay.ie": + case "ebay.it": + case "ebay.com.my": + case "ebay.nl": + case "ebay.ph": + case "ebay.pl": + case "ebay.com.sg": + case "ebay.es": + case "ebay.ch": + case "ebay.co.uk": + case "cafr.ebay.ca": + case "ebay.com": + case "community.ebay.com": + case "pages.ebay.com": + + // remove ebay tracking elements + $old_params = parse_url($url, PHP_URL_QUERY); + parse_str($old_params, $params); + + if(isset($params["mkevt"])){ unset($params["mkevt"]); } + if(isset($params["mkcid"])){ unset($params["mkcid"]); } + if(isset($params["mkrid"])){ unset($params["mkrid"]); } + if(isset($params["campid"])){ unset($params["campid"]); } + if(isset($params["customid"])){ unset($params["customid"]); } + if(isset($params["toolid"])){ unset($params["toolid"]); } + if(isset($params["_sop"])){ unset($params["_sop"]); } + if(isset($params["_dcat"])){ unset($params["_dcat"]); } + if(isset($params["epid"])){ unset($params["epid"]); } + if(isset($params["epid"])){ unset($params["oid"]); } + + $params = http_build_query($params); + + if(strlen($params) === 0){ + $replace = "\?"; + }else{ + $replace = ""; + } + + $url = preg_replace( + "/" . $replace . preg_quote($old_params, "/") . "$/", + $params, + $url + ); + break; + } + + return $url; + } + + private function number_format($number){ + + $number = explode(".", sprintf('%f', $number)); + + if(count($number) === 1){ + + return number_format((float)$number[0], 0, ",", "."); + } + + return number_format((float)$number[0], 0, ",", "") . "." . (string)$number[1]; + } + + private function bingratio($width, $height){ + + $ratio = [ + 474 / $width, + 474 / $height + ]; + + if($ratio[0] < $ratio[1]){ + + $ratio = $ratio[0]; + }else{ + + $ratio = $ratio[1]; + } + + return [ + floor($width * $ratio), + floor($height * $ratio) + ]; + } +} diff --git a/scraper/google.php b/scraper/google.php new file mode 100644 index 0000000..6a746f7 --- /dev/null +++ b/scraper/google.php @@ -0,0 +1,1562 @@ +<?php + +class google{ + + private const is_class = "."; + private const is_id = "#"; + + public function __construct(){ + + include "lib/fuckhtml.php"; + $this->fuckhtml = new fuckhtml(); + + include "lib/nextpage.php"; + $this->nextpage = new nextpage("google"); + } + + public function getfilters($page){ + + switch($page){ + + case "web": return [];/* + return [ + "country" => [ + "display" => "Country", + "option" => [ + "zz" => "Instance region", + "af" => "Afghanistan", + "al" => "Albania", + "dz" => "Algeria", + "as" => "American Samoa", + "ad" => "Andorra", + "ao" => "Angola", + "ag" => "Antigua & Barbuda", + "ar" => "Argentina", + "am" => "Armenia", + "au" => "Australia", + "at" => "Austria", + "az" => "Azerbaijan", + "bs" => "Bahamas", + "bh" => "Bahrain", + "bd" => "Bangladesh", + "by" => "Belarus", + "be" => "Belgium", + "bz" => "Belize", + "bj" => "Benin", + "bt" => "Bhutan", + "bo" => "Bolivia", + "ba" => "Bosnia & Herzegovina", + "bw" => "Botswana", + "br" => "Brazil", + "bn" => "Brunei", + "bg" => "Bulgaria", + "bf" => "Burkina Faso", + "bi" => "Burundi", + "kh" => "Cambodia", + "cm" => "Cameroon", + "ca" => "Canada", + "cv" => "Cape Verde", + "cf" => "Central African Republic", + "td" => "Chad", + "cl" => "Chile", + "co" => "Colombia", + "cg" => "Congo - Brazzaville", + "cd" => "Congo - Kinshasa", + "ck" => "Cook Islands", + "cr" => "Costa Rica", + "ci" => "Côte d’Ivoire", + "hr" => "Croatia", + "cu" => "Cuba", + "cy" => "Cyprus", + "cz" => "Czechia", + "dk" => "Denmark", + "dj" => "Djibouti", + "dm" => "Dominica", + "do" => "Dominican Republic", + "ec" => "Ecuador", + "eg" => "Egypt", + "sv" => "El Salvador", + "ee" => "Estonia", + "et" => "Ethiopia", + "fj" => "Fiji", + "fi" => "Finland", + "fr" => "France", + "ga" => "Gabon", + "gm" => "Gambia", + "ge" => "Georgia", + "de" => "Germany", + "gh" => "Ghana", + "gi" => "Gibraltar", + "gr" => "Greece", + "gl" => "Greenland", + "gt" => "Guatemala", + "gg" => "Guernsey", + "gy" => "Guyana", + "ht" => "Haiti", + "hn" => "Honduras", + "hk" => "Hong Kong", + "hu" => "Hungary", + "is" => "Iceland", + "in" => "India", + "id" => "Indonesia", + "iq" => "Iraq", + "ie" => "Ireland", + "im" => "Isle of Man", + "il" => "Israel", + "it" => "Italy", + "jm" => "Jamaica", + "jp" => "Japan", + "je" => "Jersey", + "jo" => "Jordan", + "kz" => "Kazakhstan", + "ke" => "Kenya", + "ki" => "Kiribati", + "kw" => "Kuwait", + "kg" => "Kyrgyzstan", + "la" => "Laos", + "lv" => "Latvia", + "lb" => "Lebanon", + "ls" => "Lesotho", + "ly" => "Libya", + "li" => "Liechtenstein", + "lt" => "Lithuania", + "lu" => "Luxembourg", + "mg" => "Madagascar", + "mw" => "Malawi", + "my" => "Malaysia", + "mv" => "Maldives", + "ml" => "Mali", + "mt" => "Malta", + "mu" => "Mauritius", + "mx" => "Mexico", + "fm" => "Micronesia", + "md" => "Moldova", + "mn" => "Mongolia", + "me" => "Montenegro", + "ma" => "Morocco", + "mz" => "Mozambique", + "mm" => "Myanmar (Burma)", + "na" => "Namibia", + "nr" => "Nauru", + "np" => "Nepal", + "nl" => "Netherlands", + "nz" => "New Zealand", + "ni" => "Nicaragua", + "ne" => "Niger", + "ng" => "Nigeria", + "nu" => "Niue", + "mk" => "North Macedonia", + "no" => "Norway", + "om" => "Oman", + "pk" => "Pakistan", + "ps" => "Palestine", + "pa" => "Panama", + "pg" => "Papua New Guinea", + "py" => "Paraguay", + "pe" => "Peru", + "ph" => "Philippines", + "pn" => "Pitcairn Islands", + "pl" => "Poland", + "pt" => "Portugal", + "pr" => "Puerto Rico", + "qa" => "Qatar", + "ro" => "Romania", + "ru" => "Russia", + "rw" => "Rwanda", + "ws" => "Samoa", + "sm" => "San Marino", + "st" => "São Tomé & Príncipe", + "sa" => "Saudi Arabia", + "sn" => "Senegal", + "rs" => "Serbia", + "sc" => "Seychelles", + "sl" => "Sierra Leone", + "sg" => "Singapore", + "sk" => "Slovakia", + "si" => "Slovenia", + "sb" => "Solomon Islands", + "so" => "Somalia", + "za" => "South Africa", + "kr" => "South Korea", + "es" => "Spain", + "lk" => "Sri Lanka", + "sh" => "St. Helena", + "vc" => "St. Vincent & Grenadines", + "sr" => "Suriname", + "se" => "Sweden", + "ch" => "Switzerland", + "tw" => "Taiwan", + "tj" => "Tajikistan", + "tz" => "Tanzania", + "th" => "Thailand", + "tl" => "Timor-Leste", + "tg" => "Togo", + "to" => "Tonga", + "tt" => "Trinidad & Tobago", + "tn" => "Tunisia", + "tr" => "Türkiye", + "tm" => "Turkmenistan", + "vi" => "U.S. Virgin Islands", + "ug" => "Uganda", + "ua" => "Ukraine", + "ae" => "United Arab Emirates", + "gb" => "United Kingdom", + "us" => "United States", + "uy" => "Uruguay", + "uz" => "Uzbekistan", + "vu" => "Vanuatu", + "ve" => "Venezuela", + "vn" => "Vietnam", + "zm" => "Zambia", + "zw" => "Zimbabwe" + ] + ], + "nsfw" => [ + "display" => "NSFW", + "option" => [ + "yes" => "Yes", + "no" => "No" + ] + ], + "lang" => [ // prefix with lang_ + "display" => "Language", + "option" => [ + "any" => "Any language", + "af" => "Afrikaans", + "ca" => "català", + "cs" => "čeština", + "da" => "dansk", + "de" => "Deutsch", + "et" => "eesti", + "en" => "English", + "es" => "español", + "eo" => "esperanto", + "tl" => "Filipino", + "fr" => "français", + "hr" => "hrvatski", + "id" => "Indonesia", + "is" => "íslenska", + "it" => "italiano", + "sw" => "Kiswahili", + "lv" => "latviešu", + "lt" => "lietuvių", + "hu" => "magyar", + "nl" => "Nederlands", + "no" => "norsk", + "pl" => "polski", + "pt" => "português", + "ro" => "română", + "sk" => "slovenčina", + "sl" => "slovenščina", + "fi" => "suomi", + "sv" => "svenska", + "vi" => "Tiếng Việt", + "tr" => "Türkçe", + "el" => "Ελληνικά", + "be" => "беларуская", + "bg" => "български", + "ru" => "русский", + "sr" => "српски", + "uk" => "українська", + "hy" => "հայերեն", + "iw" => "עברית", + "ar" => "العربية", + "fa" => "فارسی", + "hi" => "हिन्दी", + "th" => "ไทย", + "ko" => "한국어", + "zh-CN" => "中文 (简体)", + "zh-TW" => "中文 (繁體)", + "ja" => "日本語" + ] + ], + "time" => [ + "display" => "Time posted", + "option" => [ + "any" => "Any time", + "h" => "Last hour", + "d" => "Last 24 hours", + "w" => "Last week", + "m" => "Last month", + "y" => "Last year" + ] + ], + "verbatim" => [ + "display" => "Verbatim", + "option" => [ + "no" => "No", + "yes" => "Yes" + ] + ] + ];*/ + break; + + case "images": + return [ + "country" => [ // gl=<country> + "display" => "Country", + "option" => [ + "any" => "Instance's country", + "af" => "Afghanistan", + "al" => "Albania", + "dz" => "Algeria", + "as" => "American Samoa", + "ad" => "Andorra", + "ao" => "Angola", + "ai" => "Anguilla", + "aq" => "Antarctica", + "ag" => "Antigua and Barbuda", + "ar" => "Argentina", + "am" => "Armenia", + "aw" => "Aruba", + "au" => "Australia", + "at" => "Austria", + "az" => "Azerbaijan", + "bs" => "Bahamas", + "bh" => "Bahrain", + "bd" => "Bangladesh", + "bb" => "Barbados", + "by" => "Belarus", + "be" => "Belgium", + "bz" => "Belize", + "bj" => "Benin", + "bm" => "Bermuda", + "bt" => "Bhutan", + "bo" => "Bolivia", + "ba" => "Bosnia and Herzegovina", + "bw" => "Botswana", + "bv" => "Bouvet Island", + "br" => "Brazil", + "io" => "British Indian Ocean Territory", + "bn" => "Brunei Darussalam", + "bg" => "Bulgaria", + "bf" => "Burkina Faso", + "bi" => "Burundi", + "kh" => "Cambodia", + "cm" => "Cameroon", + "ca" => "Canada", + "cv" => "Cape Verde", + "ky" => "Cayman Islands", + "cf" => "Central African Republic", + "td" => "Chad", + "cl" => "Chile", + "cn" => "China", + "cx" => "Christmas Island", + "cc" => "Cocos (Keeling) Islands", + "co" => "Colombia", + "km" => "Comoros", + "cg" => "Congo", + "cd" => "Congo, the Democratic Republic of the", + "ck" => "Cook Islands", + "cr" => "Costa Rica", + "ci" => "Cote D'ivoire", + "hr" => "Croatia", + "cu" => "Cuba", + "cy" => "Cyprus", + "cz" => "Czech Republic", + "dk" => "Denmark", + "dj" => "Djibouti", + "dm" => "Dominica", + "do" => "Dominican Republic", + "ec" => "Ecuador", + "eg" => "Egypt", + "sv" => "El Salvador", + "gq" => "Equatorial Guinea", + "er" => "Eritrea", + "ee" => "Estonia", + "et" => "Ethiopia", + "fk" => "Falkland Islands (Malvinas)", + "fo" => "Faroe Islands", + "fj" => "Fiji", + "fi" => "Finland", + "fr" => "France", + "gf" => "French Guiana", + "pf" => "French Polynesia", + "tf" => "French Southern Territories", + "ga" => "Gabon", + "gm" => "Gambia", + "ge" => "Georgia", + "de" => "Germany", + "gh" => "Ghana", + "gi" => "Gibraltar", + "gr" => "Greece", + "gl" => "Greenland", + "gd" => "Grenada", + "gp" => "Guadeloupe", + "gu" => "Guam", + "gt" => "Guatemala", + "gn" => "Guinea", + "gw" => "Guinea-Bissau", + "gy" => "Guyana", + "ht" => "Haiti", + "hm" => "Heard Island and Mcdonald Islands", + "va" => "Holy See (Vatican City State)", + "hn" => "Honduras", + "hk" => "Hong Kong", + "hu" => "Hungary", + "is" => "Iceland", + "in" => "India", + "id" => "Indonesia", + "ir" => "Iran, Islamic Republic of", + "iq" => "Iraq", + "ie" => "Ireland", + "il" => "Israel", + "it" => "Italy", + "jm" => "Jamaica", + "jp" => "Japan", + "jo" => "Jordan", + "kz" => "Kazakhstan", + "ke" => "Kenya", + "ki" => "Kiribati", + "kp" => "Korea, Democratic People's Republic of", + "kr" => "Korea, Republic of", + "kw" => "Kuwait", + "kg" => "Kyrgyzstan", + "la" => "Lao People's Democratic Republic", + "lv" => "Latvia", + "lb" => "Lebanon", + "ls" => "Lesotho", + "lr" => "Liberia", + "ly" => "Libyan Arab Jamahiriya", + "li" => "Liechtenstein", + "lt" => "Lithuania", + "lu" => "Luxembourg", + "mo" => "Macao", + "mk" => "Macedonia, the Former Yugosalv Republic of", + "mg" => "Madagascar", + "mw" => "Malawi", + "my" => "Malaysia", + "mv" => "Maldives", + "ml" => "Mali", + "mt" => "Malta", + "mh" => "Marshall Islands", + "mq" => "Martinique", + "mr" => "Mauritania", + "mu" => "Mauritius", + "yt" => "Mayotte", + "mx" => "Mexico", + "fm" => "Micronesia, Federated States of", + "md" => "Moldova, Republic of", + "mc" => "Monaco", + "mn" => "Mongolia", + "ms" => "Montserrat", + "ma" => "Morocco", + "mz" => "Mozambique", + "mm" => "Myanmar", + "na" => "Namibia", + "nr" => "Nauru", + "np" => "Nepal", + "nl" => "Netherlands", + "an" => "Netherlands Antilles", + "nc" => "New Caledonia", + "nz" => "New Zealand", + "ni" => "Nicaragua", + "ne" => "Niger", + "ng" => "Nigeria", + "nu" => "Niue", + "nf" => "Norfolk Island", + "mp" => "Northern Mariana Islands", + "no" => "Norway", + "om" => "Oman", + "pk" => "Pakistan", + "pw" => "Palau", + "ps" => "Palestinian Territory, Occupied", + "pa" => "Panama", + "pg" => "Papua New Guinea", + "py" => "Paraguay", + "pe" => "Peru", + "ph" => "Philippines", + "pn" => "Pitcairn", + "pl" => "Poland", + "pt" => "Portugal", + "pr" => "Puerto Rico", + "qa" => "Qatar", + "re" => "Reunion", + "ro" => "Romania", + "ru" => "Russian Federation", + "rw" => "Rwanda", + "sh" => "Saint Helena", + "kn" => "Saint Kitts and Nevis", + "lc" => "Saint Lucia", + "pm" => "Saint Pierre and Miquelon", + "vc" => "Saint Vincent and the Grenadines", + "ws" => "Samoa", + "sm" => "San Marino", + "st" => "Sao Tome and Principe", + "sa" => "Saudi Arabia", + "sn" => "Senegal", + "cs" => "Serbia and Montenegro", + "sc" => "Seychelles", + "sl" => "Sierra Leone", + "sg" => "Singapore", + "sk" => "Slovakia", + "si" => "Slovenia", + "sb" => "Solomon Islands", + "so" => "Somalia", + "za" => "South Africa", + "gs" => "South Georgia and the South Sandwich Islands", + "es" => "Spain", + "lk" => "Sri Lanka", + "sd" => "Sudan", + "sr" => "Suriname", + "sj" => "Svalbard and Jan Mayen", + "sz" => "Swaziland", + "se" => "Sweden", + "ch" => "Switzerland", + "sy" => "Syrian Arab Republic", + "tw" => "Taiwan, Province of China", + "tj" => "Tajikistan", + "tz" => "Tanzania, United Republic of", + "th" => "Thailand", + "tl" => "Timor-Leste", + "tg" => "Togo", + "tk" => "Tokelau", + "to" => "Tonga", + "tt" => "Trinidad and Tobago", + "tn" => "Tunisia", + "tr" => "Turkey", + "tm" => "Turkmenistan", + "tc" => "Turks and Caicos Islands", + "tv" => "Tuvalu", + "ug" => "Uganda", + "ua" => "Ukraine", + "ae" => "United Arab Emirates", + "uk" => "United Kingdom", + "us" => "United States", + "um" => "United States Minor Outlying Islands", + "uy" => "Uruguay", + "uz" => "Uzbekistan", + "vu" => "Vanuatu", + "ve" => "Venezuela", + "vn" => "Viet Nam", + "vg" => "Virgin Islands, British", + "vi" => "Virgin Islands, U.S.", + "wf" => "Wallis and Futuna", + "eh" => "Western Sahara", + "ye" => "Yemen", + "zm" => "Zambia", + "zw" => "Zimbabwe" + ] + ], + "nsfw" => [ + "display" => "NSFW", + "option" => [ + "yes" => "Yes", // safe=active + "no" => "No" // safe=off + ] + ], + "lang" => [ // lr=<lang> (prefix lang with "lang_") + "display" => "Language", + "option" => [ + "any" => "Any language", + "ar" => "Arabic", + "bg" => "Bulgarian", + "ca" => "Catalan", + "cs" => "Czech", + "da" => "Danish", + "de" => "German", + "el" => "Greek", + "en" => "English", + "es" => "Spanish", + "et" => "Estonian", + "fi" => "Finnish", + "fr" => "French", + "hr" => "Croatian", + "hu" => "Hungarian", + "id" => "Indonesian", + "is" => "Icelandic", + "it" => "Italian", + "iw" => "Hebrew", + "ja" => "Japanese", + "ko" => "Korean", + "lt" => "Lithuanian", + "lv" => "Latvian", + "nl" => "Dutch", + "no" => "Norwegian", + "pl" => "Polish", + "pt" => "Portuguese", + "ro" => "Romanian", + "ru" => "Russian", + "sk" => "Slovak", + "sl" => "Slovenian", + "sr" => "Serbian", + "sv" => "Swedish", + "tr" => "Turkish", + "zh-CN" => "Chinese (Simplified)", + "zh-TW" => "Chinese (Traditional)" + ] + ], + "newer" => [ // &sort=review-date:r:20090301:20090430 + "display" => "Newer than", + "option" => "_DATE" + ], + "older" => [ + "display" => "Older than", + "option" => "_DATE" + ], + "size" => [ // tbs=isz:<size> + "display" => "Size", + "option" => [ + "any" => "Any size", + "l" => "Large", + "m" => "Medium", + "i" => "Icon" + ] + ], + "color" => [ // tbs=ic:<color> + "display" => "Color", + "option" => [ + "any" => "Any color", + "gray" => "Black and white", + "trans" => "Transparent", + // from here, format is + // tbs=specific,isc:<color> + "red" => "Red", + "orange" => "Orange", + "yellow" => "Yellow", + "green" => "Green", + "teal" => "Teal", + "blue" => "Blue", + "purple" => "Purple", + "pink" => "Pink", + "white" => "White", + "gray" => "Gray", + "black" => "Black", + "brown" => "Brown" + ] + ], + "type" => [ // tbs=itp:<type> + "display" => "Type", + "option" => [ + "any" => "Any type", + "clipart" => "Clip Art", + "lineart" => "Line Drawing", + "animated" => "GIF" + ] + ], + "rights" => [ // tbs=il:<rights> + "display" => "Usage rights", + "option" => [ + "any" => "No license", + "cl" => "Creative Commons licenses", + "ol" => "Commercial & other licenses" + ] + ] + ]; + break; + } + } + + private function get($url, $get = []){ + + $headers = [ + "User-Agent: Mozilla/5.0 (Linux; U; Android 2.3.3; pt-pt; LG-P500h-parrot Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1 MMS/LG-Android-MMS-V1.0/1.2", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip", + "DNT: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1" + ]; + + $curlproc = curl_init(); + + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + + curl_setopt($curlproc, CURLOPT_URL, $url); + + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + public function web($get){ + + $handle = fopen("scraper/google.html", "r"); + $html = fread($handle, filesize("scraper/google.html")); + fclose($handle); + + $this->fuckhtml->load($html); + + $out = [ + "status" => "ok", + "spelling" => [ + "type" => "no_correction", + "using" => null, + "correction" => null + ], + "npt" => null, + "answer" => [], + "web" => [], + "image" => [], + "video" => [], + "news" => [], + "related" => [] + ]; + + $styles = + $this->fuckhtml + ->getElementsByTagName("style"); + + $this->computedstyle = []; + + foreach($styles as $style){ + + $this->computedstyle = + array_merge( + $this->computedstyle, + $this->parsestyles($style["innerHTML"]) + ); + } + + // get images in javascript var + preg_match( + '/google\.ldi=({[^}]+})/', + $html, + $js_image + ); + + if(count($js_image) !== 0){ + + $js_image = json_decode($js_image[1], true); + }else{ + + $js_image = []; + } + + // get nodes + // fuck you google!!!!!!!!!!!!!! + + $containers = + $this->fuckhtml + ->getElementsByClassName( + $this->findstyles( + [ + "background-color" => "#fff", + "margin-bottom" => "10px", + "-webkit-box-shadow" => "0 1px 6px rgba(32,33,36,0.28)", + "border-radius" => "8px" + ], + self::is_class + ), + "div" + ); + + foreach($containers as $container){ + + $this->fuckhtml->load($container); + + // get link at the top + $link = + $this->fuckhtml + ->getElementsByTagName( + "a" + ); + + if(count($link) !== 0){ + + $link = + $this->decodeurl( + $link + [0] + ["attributes"] + ["href"] + ); + } + + /* + Check for carousel presence + */ + $carousel = + $this->fuckhtml + ->getElementsByClassName( + "pcitem", + "div" + ); + + $title = + $this->fuckhtml + ->getElementsByClassName( + $this->findstyles( + [ + "color" => "#1967d2", + "font-size" => "20px", + "line-height" => "26px" + ], + self::is_class + ), + "div" + ); + + $carousel_title = + $this->fuckhtml + ->getElementsByClassName( + $this->findstyles( + [ + "font-size" => "16px", + "line-height" => "20px", + "font-weight" => "400" + ], + self::is_class + ), + "div" + ); + + if(count($carousel) !== 0){ + + $sublink = []; // twitter carousel sublinks + foreach($carousel as $item){ + + $this->fuckhtml->load($item); + + $url = + $this->decodeurl( + $this->fuckhtml + ->getElementsByTagName( + "a" + )[0] + ["attributes"] + ["href"] + ); + + // detect if its a twitter carousel or + // a list of news articles + + $grey_node = + $this->fuckhtml + ->getElementsByClassName( + $this->findstyles( + [ + "white-space" => "pre-line", + "word-wrap" => "break-word" + ], + self::is_class + ), + "div" + ); + + if(count($carousel_title) !== 0){ + + if( + $this->fuckhtml + ->getTextContent( + $carousel_title[0] + ) + == "Top stories" + ){ + + $img = + $this->fuckhtml + ->getElementsByTagName("img"); + + if( + count($img) !== 0 && + isset($img[0]["attributes"]["id"]) && + isset($js_image[$img[0]["attributes"]["id"]]) + ){ + + $img = [ + "url" => $js_image[$img[0]["attributes"]["id"]], + "ratio" => "16:9" + ]; + }else{ + + $img = [ + "url" => null, + "ratio" => null + ]; + } + + /* + Is a news node + */ + $out["news"][] = [ + "title" => + $this->fuckhtml + ->getTextContent( + $grey_node[0] + ), + "description" => null, + "date" => + strtotime( + explode( + "\n", + $grey_node[1]["innerHTML"] + )[1] + ), + "thumb" => $img, + "url" => $url + ]; + } + }else{ + + /* + Is a web node (twitter-like) + create a link -> sublink structure and + ignore images + */ + + switch(count($grey_node)){ + + case 0: + continue 2; + + case 1: + $sublink_title = $grey_node[0]; + $sublink_description = null; + break; + + case 2: + $sublink_title = $grey_node[1]; + $sublink_description = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $grey_node[0] + ) + ); + break; + } + + $sublink_url = + $this->decodeurl( + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByTagName( + "a" + )[0] + ["attributes"] + ["href"] + ) + ); + + if($link == $sublink_url){ + + continue; + } + + $sublink_title = + explode( + " • ", + $this->fuckhtml + ->getTextContent( + $sublink_title["innerHTML"] + ) + ); + + if(count($sublink_title) !== 1){ + + $date = strtotime($sublink_title[1]); + }else{ + + $date = null; + } + + $sublink_title = $this->titledots($sublink_title[0]); + + $sublink[] = [ + "title" => $sublink_title, + "date" => $date, + "description" => $sublink_description, + "url" => $sublink_url + ]; + } + } + + // if it was a web node + if(count($sublink) !== 0){ + + $out["web"][] = [ + "title" => + $this->titledots( + $this->fuckhtml + ->getTextContent( + $title[0] + ) + ), + "description" => null, + "url" => $url, + "date" => null, + "type" => "web", + "thumb" => [ + "url" => null, + "ratio" => null + ], + "sublink" => $sublink, + "table" => [] + ]; + } + + continue; + } + + if(count($title) !== 0){ + + /* + Get WEB search results + */ + + $thumb = + $this->fuckhtml + ->getElementsByTagName("img"); + + if( + count($thumb) !== 0 && + isset($js_image[$thumb[0]["attributes"]["id"]]) + ){ + + $thumb = [ + "url" => + $js_image[$thumb[0]["attributes"]["id"]], + "ratio" => "1:1" + ]; + }else{ + + $thumb = [ + "url" => null, + "ratio" => null + ]; + } + + // this contains description, sublinks + $inner_category = + $this->fuckhtml + ->getElementsByClassName( + $this->findstyles( + [ + "white-space" => "pre-line", + "word-wrap" => "break-word" + ], + self::is_class + ), + "div" + ); + + // set empty values + $description = null; + $table = []; + $sublinks = []; + $date = null; + + foreach($inner_category as $category){ + + if($category["level"] !== 6){ + + // enterring protocol 6 + // and u dont seem to understaaaaandddddd + continue; + } + + $this->fuckhtml->load($category); + + // check if its a table + preg_match( + '/^[A-z0-9 ]+: <span/', + $category["innerHTML"], + $tablematch + ); + + if(count($tablematch) !== 0){ + + $categories = explode("<br>", $category["innerHTML"]); + + foreach($categories as $cat){ + + $cat = explode(":", $cat, 2); + + $table[ + $this->fuckhtml + ->getTextContent( + $cat[0] + ) + ] = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $cat[1] + ) + ); + } + continue; + } + + $spans = + $this->fuckhtml + ->getElementsByTagName("span"); + + foreach($spans as $span){ + + // replace element with nothing + if(empty($description)){ + $category["innerHTML"] = + str_replace( + $span["outerHTML"], + "", + $category["innerHTML"] + ); + } + + // get rating + if(isset($span["attributes"]["aria-hidden"])){ + + $table["Rating"] = $span["innerHTML"]; + continue; + } + } + + if(empty($description)){ + + $description = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $category + ) + ); + } + } + + // check if traversed div is the description + /* + if( + count( + $this->fuckhtml + ->getElementsByTagName("*") + ) === 0 + ){ + + $description = + $this->fuckhtml + ->getTextContent($inner_category); + }else{ + + $this-> + + // we need to traverse description struct + foreach($inner_category as $category){ + + // detect description + $this->fuckhtml->load($category); + + $spans = + $this->fuckhtml + ->getElementsByTagName("span"); + + $is_desc = false; + $is_first_span = true; + + foreach($spans as $span){ + + // get rating + if(isset($span["attributes"]["aria-hidden"])){ + + $table["Rating"] = $span["innerHTML"] . "/5"; + continue; + } + + // get date posted + if( + $is_first_span && + $date_tmp = strtotime($span["innerHTML"]) + ){ + + $date = $date_tmp; + continue; + } + + $is_first_span = false; + } + } + }*/ + + // get sublinks + $this->fuckhtml->load($container["innerHTML"]); + + $as = + $this->fuckhtml->getElementsByTagName("a"); + + foreach($as as $a){ + + $this->fuckhtml->load($a); + + $detect = + $this->fuckhtml + ->getElementsByClassName( + $this->findstyles( + [ + "color" => "#1967d2", + "font-size" => "14px", + "line-height" => "20px" + ], + self::is_class + ), + "span" + ); + + if(count($detect) !== 0){ + + $sublinks[] = [ + "title" => + $this->titledots( + $this->fuckhtml + ->getTextContent( + $a + ) + ), + "date" => null, + "description" => null, + "url" => + $this->decodeurl( + $a["attributes"]["href"] + ) + ]; + } + } + + $data = [ + "title" => + $this->titledots( + $this->fuckhtml + ->getTextContent( + $title[0] + ) + ), + "description" => $description, + "url" => $link, + "date" => $date, + "type" => "web", + "thumb" => $thumb, + "sublink" => $sublinks, + "table" => $table + ]; + + $out["web"][] = $data; + + continue; + } + + /* + Check related searches node + */ + $relateds = + $this->fuckhtml + ->getElementsByClassName( + $this->findstyles( + [ + "display" => "block", + "position" => "relative", + "width" => "100%" + ], + self::is_class + ), + "a" + ); + + if(count($relateds) !== 0){ + + foreach($relateds as $related){ + + $out["related"][] = + $this->fuckhtml + ->getTextContent( + $related + ); + } + } + + /* + Get next page + */ + $nextpage = + $this->fuckhtml + ->getElementsByClassName( + $this->findstyles( + [ + "-webkit-box-flex" => "1", + "display" => "block" + ], + self::is_class + ), + "a" + ); + + if(count($nextpage) !== 0){ + + $out["npt"] = + explode( + "?", + $this->fuckhtml + ->getTextContent( + $nextpage[0] + ["attributes"] + ["href"] + ) + )[1]; + } + } + + return $out; + } + + public function image($get){ + + $handle = fopen("scraper/google-img.html", "r"); + $html = fread($handle, filesize("scraper/google-img.html")); + fclose($handle); + + $this->fuckhtml->load($html); + + $out = [ + "status" => "ok", + "npt" => null, + "image" => [] + ]; + + $images = + $this->fuckhtml + ->getElementsByClassName( + "islrtb isv-r", + "div" + ); + + // get next page + // https://www.google.com/search + // ?q=higurashi + // &tbm=isch + // &async=_id%3Aislrg_c%2C_fmt%3Ahtml + // &asearch=ichunklite + // &ved=0ahUKEwidjYXJqJSAAxWrElkFHZ07CDwQtDIIQygA + $ved = + $this->fuckhtml + ->getElementById("islrg", "div"); + + if($ved){ + + $ved = + $this->fuckhtml + ->getTextContent( + $ved["attributes"]["data-ved"] + ); + + // &vet=1{$ved}..i (10ahUKEwidjYXJqJSAAxWrElkFHZ07CDwQtDIIQygA..i) + + /* + These 2 are handled by us + start = start + number of results + ijn = current page number + */ + // &start=100 + // &ijn=1 + + // &imgvl=CAEY7gQgBSj3Aji8VTjXVUC4AUC3AUgAYNdV + preg_match( + '/var e=\'([A-z0-9]+)\';/', + $html, + $imgvl + ); + + $imgvl = $imgvl[1]; + + $out["npt"] = [ + "q" => $get["s"], + "tbm" => "isch", + "async" => "_id:islrg_c,_fmt:html", + "asearch" => "ichunklite", + "ved" => $ved, + "vet" => "1" . $ved . "..i", + "start" => 100, + "ijn" => 1, + "imgvl" => $imgvl + ]; + } + + foreach($images as $image){ + + $this->fuckhtml->load($image); + $img = + $this->fuckhtml + ->getElementsByTagName("img")[0]; + + $og_width = (int)$image["attributes"]["data-ow"]; + $og_height = (int)$image["attributes"]["data-oh"]; + $thumb_width = (int)$image["attributes"]["data-tw"]; + + $ratio = $og_width / $og_height; + + if(isset($img["attributes"]["data-src"])){ + + $src = &$img["attributes"]["data-src"]; + }else{ + + $src = &$img["attributes"]["src"]; + } + + $thumb_height = floor($thumb_width / $ratio); + + $out["image"][] = [ + "title" => + $this->titledots( + $this->fuckhtml + ->getTextContent( + $image["attributes"]["data-pt"] + ) + ), + "source" => [ + [ + "url" => + $this->fuckhtml + ->getTextContent( + $image["attributes"]["data-ou"] + ), + "width" => $og_width, + "height" => $og_height + ], + [ + "url" => + $this->fuckhtml + ->getTextContent( + $src + ), + "width" => $thumb_width, + "height" => $thumb_height + ] + ], + "url" => + $this->fuckhtml + ->getTextContent( + $image["attributes"]["data-ru"] + ) + ]; + } + + return $out; + } + + private function findstyles($rules, $is){ + + ksort($rules); + + foreach($this->computedstyle as $stylename => $styles){ + + if($styles == $rules){ + + preg_match( + '/\\' . $is . '([^ .]+)/', + $stylename, + $out + ); + + if(count($out) === 2){ + + return $out[1]; + } + + return false; + } + } + + return false; + } + + private function parsestyles($style){ + + // get style tags + preg_match_all( + '/([^{]+){([^}]+)}/', + $style, + $tags_regex + ); + + $tags = []; + + for($i=0; $i<count($tags_regex[0]); $i++){ + + $tagnames = explode(",", trim($tags_regex[1][$i])); + + foreach($tagnames as $tagname){ + + $tagname = trim($tagname); + + if(!isset($tags[$tagname])){ + $tags[$tagname] = []; + } + + $values = explode(";", $tags_regex[2][$i]); + + foreach($values as $value){ + + $value = explode(":", $value, 2); + + if(count($value) !== 2){ + + continue; + } + + $tags[$tagname][trim($value[0])] = + trim($value[1]); + } + } + } + + foreach($tags as &$value){ + + ksort($value); + } + + return $tags; + } + + private function decodeurl($url){ + + preg_match( + '/^\/url\?q=([^&]+)|^\/interstitial\?url=([^&]+)/', + $this->fuckhtml + ->getTextContent($url), + $match + ); + + if(count($match) !== 0){ + + if(!empty($match[1])){ + + return urldecode($match[1]); + } + + if(!empty($match[2])){ + + return urldecode($match[2]); + } + } + + return null; + } + + private function titledots($title){ + + return rtrim($title, ".… \t\n\r\0\x0B"); + } +} + diff --git a/scraper/marginalia.php b/scraper/marginalia.php new file mode 100644 index 0000000..c8ab09f --- /dev/null +++ b/scraper/marginalia.php @@ -0,0 +1,242 @@ +<?php + +class marginalia{ + public function __construct(){ + + $this->key = "public"; + } + + public function getfilters($page){ + + switch($page){ + + case "web": + return [ + "profile" => [ + "display" => "Profile", + "option" => [ + "any" => "Default", + "modern" => "Modern" + ] + ], + "format" => [ + "display" => "Format", + "option" => [ + "any" => "Any", + "html5" => "html5", + "xhtml" => "xhtml", + "html123" => "html123" + ] + ], + "file" => [ + "display" => "File", + "option" => [ + "any" => "Any", + "nomedia" => "Deny media", + "media" => "Contains media", + "audio" => "Contains audio", + "video" => "Contains video", + "archive" => "Contains archive", + "document" => "Contains document" + ] + ], + "javascript" => [ + "display" => "Javascript", + "option" => [ + "any" => "Allow JS", + "deny" => "Deny JS", + "require" => "Require JS" + ] + ], + "trackers" => [ + "display" => "Trackers", + "option" => [ + "any" => "Allow trackers", + "deny" => "Deny trackers", + "require" => "Require trackers" + ] + ], + "cookies" => [ + "display" => "Cookies", + "option" => [ + "any" => "Allow cookies", + "deny" => "Deny cookies", + "require" => "Require cookies" + ] + ], + "affiliate" => [ + "display" => "Affiliate links in body", + "option" => [ + "any" => "Allow affiliate links", + "deny" => "Deny affiliate links", + "require" => "Require affiliate links" + ] + ] + ]; + } + } + + private function get($url, $get = []){ + + $headers = [ + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip", + "DNT: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1" + ]; + + $curlproc = curl_init(); + + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + + curl_setopt($curlproc, CURLOPT_URL, $url); + + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + public function web($get){ + + $search = [$get["s"]]; + $profile = $get["profile"]; + $format = $get["format"]; + $file = $get["file"]; + + foreach( + [ + "javascript" => $get["javascript"], + "trackers" => $get["trackers"], + "cookies" => $get["cookies"], + "affiliate" => $get["affiliate"] + ] + as $key => $value + ){ + + if($value == "any"){ continue; } + + switch($key){ + + case "javascript": $str = "js:true"; break; + case "trackers": $str = "special:tracking"; break; + case "cookies": $str = "special:cookies"; break; + case "affiliate": $str = "special:affiliate"; break; + } + + if($value == "deny"){ + $str = "-" . $str; + } + + $search[] = $str; + } + + if($format != "any"){ + + $search[] = "format:$format"; + } + + switch($file){ + + case "any": break; + case "nomedia": $search[] = "-special:media"; break; + case "media": $search[] = "special:media"; break; + + default: + $search[] = "file:$file"; + } + + $search = implode(" ", $search); + + $params = [ + "count" => 20 + ]; + + if($profile == "modern"){ + + $params["index"] = 1; + } + + try{ + $json = + $this->get( + "https://api.marginalia.nu/{$this->key}/search/" . urlencode($search), + $params + ); + }catch(Exception $error){ + + throw new Exception("Failed to get JSON"); + } + + if($json == "Slow down"){ + + throw new Exception("The API key used is rate limited. Please try again in a few minutes."); + } + + $json = json_decode($json, true); + /* + $handle = fopen("scraper/marginalia.json", "r"); + $json = json_decode(fread($handle, filesize("scraper/marginalia.json")), true); + fclose($handle);*/ + + $out = [ + "status" => "ok", + "spelling" => [ + "type" => "no_correction", + "using" => null, + "correction" => null + ], + "npt" => null, + "answer" => [], + "web" => [], + "image" => [], + "video" => [], + "news" => [], + "related" => [] + ]; + + foreach($json["results"] as $result){ + + $out["web"][] = [ + "title" => $result["title"], + "description" => str_replace("\n", " ", $result["description"]), + "url" => $result["url"], + "date" => null, + "type" => "web", + "thumb" => [ + "url" => null, + "ratio" => null + ], + "sublink" => [], + "table" => [] + ]; + } + + return $out; + } +} + diff --git a/scraper/mojeek.php b/scraper/mojeek.php new file mode 100644 index 0000000..a0b5016 --- /dev/null +++ b/scraper/mojeek.php @@ -0,0 +1,1182 @@ +<?php + +class mojeek{ + public function __construct(){ + + include "lib/fuckhtml.php"; + $this->fuckhtml = new fuckhtml(); + + include "lib/nextpage.php"; + $this->nextpage = new nextpage("mojeek"); + } + + public function getfilters($page){ + + switch($page){ + + case "web": + return [ + "focus" => [ + "display" => "Focus", + "option" => [ + "any" => "No focus", + "blogs" => "Blogs", + "Dictionary" => "Dictionary", + "Recipes" => "Recipes", + "Time" => "Time", + "Weather" => "Weather" + ] + ], + "lang" => [ + "display" => "Language", + "option" => [ + "any" => "Any language", + "af" => "Afrikaans", + "sq" => "Albanian", + "an" => "Aragonese", + "ay" => "Aymara", + "bi" => "Bislama", + "br" => "Breton", + "ca" => "Catalan", + "kw" => "Cornish", + "co" => "Corsican", + "hr" => "Croatian", + "da" => "Danish", + "nl" => "Dutch", + "dz" => "Dzongkha", + "en" => "English", + "fj" => "Fijian", + "fi" => "Finnish", + "fr" => "French", + "gd" => "Gaelic", + "gl" => "Galician", + "de" => "German", + "ht" => "Haitian", + "io" => "Ido", + "id" => "Indonesian", + "ia" => "Interlingua", + "ie" => "Interlingue", + "ga" => "Irish", + "it" => "Italian", + "rw" => "Kinyarwanda", + "la" => "Latin", + "li" => "Limburgish", + "lb" => "Luxembourgish", + "no" => "Norwegian", + "nb" => "Norwegian Bokmål", + "nn" => "Norwegian Nynorsk", + "oc" => "Occitan (post 1500)", + "pl" => "Polish", + "pt" => "Portuguese", + "rm" => "Romansh", + "rn" => "Rundi", + "sg" => "Sango", + "so" => "Somali", + "es" => "Spanish", + "sw" => "Swahili", + "ss" => "Swati", + "sv" => "Swedish", + "ty" => "Tahitian", + "to" => "Tonga (Tonga Islands)", + "ts" => "Tsonga", + "vo" => "Volapük", + "wa" => "Walloon", + "cy" => "Welsh", + "xh" => "Xhosa", + "zu" => "Zulu" + ] + ], + "country" => [ + "display" => "Country", + "option" => [ + "any" => "No location bias", + "af" => "Afghanistan", + "ax" => "Åland Islands", + "al" => "Albania", + "dz" => "Algeria", + "as" => "American Samoa", + "ad" => "Andorra", + "ao" => "Angola", + "ai" => "Anguilla", + "aq" => "Antarctica", + "ag" => "Antigua and Barbuda", + "ar" => "Argentina", + "am" => "Armenia", + "aw" => "Aruba", + "au" => "Australia", + "at" => "Austria", + "az" => "Azerbaijan", + "bs" => "Bahamas", + "bh" => "Bahrain", + "bd" => "Bangladesh", + "bb" => "Barbados", + "by" => "Belarus", + "be" => "Belgium", + "bz" => "Belize", + "bj" => "Benin", + "bm" => "Bermuda", + "bt" => "Bhutan", + "bo" => "Bolivia (Plurinational State of)", + "bq" => "Bonaire, Sint Eustatius and Saba", + "ba" => "Bosnia and Herzegovina", + "bw" => "Botswana", + "bv" => "Bouvet Island", + "br" => "Brazil", + "io" => "British Indian Ocean Territory", + "bn" => "Brunei Darussalam", + "bg" => "Bulgaria", + "bf" => "Burkina Faso", + "bi" => "Burundi", + "cv" => "Cabo Verde", + "kh" => "Cambodia", + "cm" => "Cameroon", + "ca" => "Canada", + "ky" => "Cayman Islands", + "cf" => "Central African Republic", + "td" => "Chad", + "cl" => "Chile", + "cn" => "China", + "cx" => "Christmas Island", + "cc" => "Cocos (Keeling) Islands", + "co" => "Colombia", + "km" => "Comoros", + "cg" => "Congo", + "cd" => "Congo (Democratic Republic of the)", + "ck" => "Cook Islands", + "cr" => "Costa Rica", + "ci" => "Côte d'Ivoire", + "hr" => "Croatia", + "cu" => "Cuba", + "cw" => "Curaçao", + "cy" => "Cyprus", + "cz" => "Czechia", + "dk" => "Denmark", + "dj" => "Djibouti", + "dm" => "Dominica", + "do" => "Dominican Republic", + "ec" => "Ecuador", + "eg" => "Egypt", + "sv" => "El Salvador", + "gq" => "Equatorial Guinea", + "er" => "Eritrea", + "ee" => "Estonia", + "et" => "Ethiopia", + "fk" => "Falkland Islands (Malvinas)", + "fo" => "Faroe Islands", + "fj" => "Fiji", + "fi" => "Finland", + "fr" => "France", + "gf" => "French Guiana", + "pf" => "French Polynesia", + "tf" => "French Southern Territories", + "ga" => "Gabon", + "gm" => "Gambia", + "ge" => "Georgia", + "de" => "Germany", + "gh" => "Ghana", + "gi" => "Gibraltar", + "gr" => "Greece", + "gl" => "Greenland", + "gd" => "Grenada", + "gp" => "Guadeloupe", + "gu" => "Guam", + "gt" => "Guatemala", + "gg" => "Guernsey", + "gn" => "Guinea", + "gw" => "Guinea-Bissau", + "gy" => "Guyana", + "ht" => "Haiti", + "hm" => "Heard Island and McDonald Islands", + "va" => "Holy See", + "hn" => "Honduras", + "hk" => "Hong Kong", + "hu" => "Hungary", + "is" => "Iceland", + "in" => "India", + "id" => "Indonesia", + "ir" => "Iran (Islamic Republic of)", + "iq" => "Iraq", + "ie" => "Ireland", + "im" => "Isle of Man", + "il" => "Israel", + "it" => "Italy", + "jm" => "Jamaica", + "jp" => "Japan", + "je" => "Jersey", + "jo" => "Jordan", + "kz" => "Kazakhstan", + "ke" => "Kenya", + "ki" => "Kiribati", + "kp" => "Korea (Democratic People's Republic of)", + "kr" => "Korea (Republic of)", + "kw" => "Kuwait", + "kg" => "Kyrgyzstan", + "la" => "Lao People's Democratic Republic", + "lv" => "Latvia", + "lb" => "Lebanon", + "ls" => "Lesotho", + "lr" => "Liberia", + "ly" => "Libya", + "li" => "Liechtenstein", + "lt" => "Lithuania", + "lu" => "Luxembourg", + "mo" => "Macao", + "mk" => "Macedonia (the former Yugoslav Republic of)", + "mg" => "Madagascar", + "mw" => "Malawi", + "my" => "Malaysia", + "mv" => "Maldives", + "ml" => "Mali", + "mt" => "Malta", + "mh" => "Marshall Islands", + "mq" => "Martinique", + "mr" => "Mauritania", + "mu" => "Mauritius", + "yt" => "Mayotte", + "mx" => "Mexico", + "fm" => "Micronesia (Federated States of)", + "md" => "Moldova (Republic of)", + "mc" => "Monaco", + "mn" => "Mongolia", + "me" => "Montenegro", + "ms" => "Montserrat", + "ma" => "Morocco", + "mz" => "Mozambique", + "mm" => "Myanmar", + "na" => "Namibia", + "nr" => "Nauru", + "np" => "Nepal", + "nl" => "Netherlands", + "nc" => "New Caledonia", + "nz" => "New Zealand", + "ni" => "Nicaragua", + "ne" => "Niger", + "ng" => "Nigeria", + "nu" => "Niue", + "nf" => "Norfolk Island", + "mp" => "Northern Mariana Islands", + "no" => "Norway", + "om" => "Oman", + "pk" => "Pakistan", + "pw" => "Palau", + "ps" => "Palestine, State of", + "pa" => "Panama", + "pg" => "Papua New Guinea", + "py" => "Paraguay", + "pe" => "Peru", + "ph" => "Philippines", + "pn" => "Pitcairn", + "pl" => "Poland", + "pt" => "Portugal", + "pr" => "Puerto Rico", + "qa" => "Qatar", + "re" => "Réunion", + "ro" => "Romania", + "ru" => "Russian Federation", + "rw" => "Rwanda", + "bl" => "Saint Barthélemy", + "sh" => "Saint Helena, Ascension and Tristan da Cunha", + "kn" => "Saint Kitts and Nevis", + "lc" => "Saint Lucia", + "mf" => "Saint Martin (French part)", + "pm" => "Saint Pierre and Miquelon", + "vc" => "Saint Vincent and the Grenadines", + "ws" => "Samoa", + "sm" => "San Marino", + "st" => "Sao Tome and Principe", + "sa" => "Saudi Arabia", + "sn" => "Senegal", + "rs" => "Serbia", + "sc" => "Seychelles", + "sl" => "Sierra Leone", + "sg" => "Singapore", + "sx" => "Sint Maarten (Dutch part)", + "sk" => "Slovakia", + "si" => "Slovenia", + "sb" => "Solomon Islands", + "so" => "Somalia", + "za" => "South Africa", + "gs" => "South Georgia and South Sandwich Islands", + "ss" => "South Sudan", + "es" => "Spain", + "lk" => "Sri Lanka", + "sd" => "Sudan", + "sr" => "Suriname", + "sj" => "Svalbard and Jan Mayen", + "sz" => "Swaziland", + "se" => "Sweden", + "ch" => "Switzerland", + "sy" => "Syrian Arab Republic", + "tw" => "Taiwan", + "tj" => "Tajikistan", + "tz" => "Tanzania, United Republic of", + "th" => "Thailand", + "tl" => "Timor-Leste", + "tg" => "Togo", + "tk" => "Tokelau", + "to" => "Tonga", + "tt" => "Trinidad and Tobago", + "tn" => "Tunisia", + "tr" => "Turkey", + "tm" => "Turkmenistan", + "tc" => "Turks and Caicos Islands", + "tv" => "Tuvalu", + "ug" => "Uganda", + "ua" => "Ukraine", + "ae" => "United Arab Emirates", + "gb" => "United Kingdom", + "us" => "United States of America", + "um" => "United States Minor Outlying Islands", + "uy" => "Uruguay", + "uz" => "Uzbekistan", + "vu" => "Vanuatu", + "ve" => "Venezuela (Bolivarian Republic of)", + "vn" => "Viet Nam", + "vg" => "Virgin Islands (British)", + "vi" => "Virgin Islands (U.S.)", + "wf" => "Wallis and Futuna", + "eh" => "Western Sahara", + "ye" => "Yemen", + "zm" => "Zambia", + "zw" => "Zimbabwe" + ] + ], + "region" => [ + "display" => "Region", + "option" => [ + "any" => "Any region", + "eu" => "European Union", + "de" => "Germany", + "fr" => "France", + "uk" => "United Kingdom" + ] + ], + "domain" => [ + "display" => "Results per domain", + "option" => [ + "1" => "1 result", + "2" => "2 results", + "3" => "3 results", + "4" => "4 results", + "5" => "5 results", + "10" => "10 results", + "0" => "Unlimited", + ] + ] + ]; + break; + + case "news": + return []; + } + } + + private function get($url, $get = []){ + + $headers = [ + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip", + "DNT: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1" + ]; + + $curlproc = curl_init(); + + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + + curl_setopt($curlproc, CURLOPT_URL, $url); + + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + public function web($get){ + + if($get["npt"]){ + + $token = $this->nextpage->get($get["npt"], "web"); + + try{ + $html = + $this->get( + "https://www.mojeek.com" . $token, + [] + ); + }catch(Exception $error){ + + throw new Exception("Failed to get HTML"); + } + + }else{ + $search = $get["s"]; + $lang = $get["lang"]; + $country = $get["country"]; + $region = $get["region"]; + $domain = $get["domain"]; + $focus = $get["focus"]; + + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $params = [ + "q" => $search, + "t" => 20, // number of results/page + "tn" => 7, // number of news results/page + "date" => 1, // show date + "tlen" => 128, // max length of title + "dlen" => 511, // max length of description + "arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect! + ]; + + switch($focus){ + + case "any": break; + + case "blogs": + $params["fmt"] = "sst"; + $params["sst"] = "1"; + break; + + default: + $params["foc_t"] = $focus; + break; + } + + if($lang != "any"){ + + $params["lb"] = $lang; + } + + if($region != "any"){ + + $params["reg"] = $region; + } + + if($domain != "1"){ + + $params["si"] = $domain; + } + + try{ + $html = + $this->get( + "https://www.mojeek.com/search", + $params + ); + }catch(Exception $error){ + + throw new Exception("Failed to get HTML"); + } + /* + $handle = fopen("scraper/mojeek.html", "r"); + $html = fread($handle, filesize("scraper/mojeek.html")); + fclose($handle);*/ + + } + + $out = [ + "status" => "ok", + "spelling" => [ + "type" => "no_correction", + "using" => null, + "correction" => null + ], + "npt" => null, + "answer" => [], + "web" => [], + "image" => [], + "video" => [], + "news" => [], + "related" => [] + ]; + + $this->fuckhtml->load($html); + + $results = + $this->fuckhtml + ->getElementsByClassName("results-standard", "ul"); + + if(count($results) === 0){ + + return $out; + } + + $this->fuckhtml->load($results[0]); + + /* + Get search results + */ + $results = + $this->fuckhtml + ->getElementsByTagName("li"); + + foreach($results as $result){ + + $data = [ + "title" => null, + "description" => null, + "url" => null, + "date" => null, + "type" => "web", + "thumb" => [ + "url" => null, + "ratio" => null + ], + "sublink" => [], + "table" => [] + ]; + + $this->fuckhtml->load($result); + + $title = + $this->fuckhtml + ->getElementsByClassName("title", "a")[0]; + + $data["title"] = + html_entity_decode( + $this->fuckhtml + ->getTextContent( + $title["innerHTML"] + ) + ); + + $data["url"] = + html_entity_decode( + $this->fuckhtml + ->getTextContent( + $title["attributes"]["href"] + ) + ); + + $description = + $this->fuckhtml + ->getElementsByClassName( + "s", "p" + ); + + if(count($description) !== 0){ + + $data["description"] = + $this->titledots( + html_entity_decode( + $this->fuckhtml + ->getTextContent( + $description[0] + ) + ) + ); + } + + $data["date"] = + explode( + " - ", + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName("i", "p")[1] + ) + ); + + $data["date"] = + strtotime( + $data["date"][count($data["date"]) - 1] + ); + + $out["web"][] = $data; + } + + /* + Get instant answers + */ + $this->fuckhtml->load($html); + + $infoboxes = + $this->fuckhtml + ->getElementsByClassName( + "infobox infobox-top", + "div" + ); + + foreach($infoboxes as $infobox){ + + $answer = [ + "title" => null, + "description" => [], + "url" => null, + "thumb" => null, + "table" => [], + "sublink" => [] + ]; + + // load first part with title + short definition + $infobox_html = + explode( + "<hr>", + $infobox["innerHTML"] + ); + + $this->fuckhtml->load($infobox_html[0]); + + // title + $answer["title"] = + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByTagName("h1")[0] + ); + + // short definition + $definition = + $this->fuckhtml + ->getElementsByTagName( + "p" + ); + + if(count($definition) !== 0){ + + $answer["description"][] = [ + "type" => "quote", + "value" => + $this->fuckhtml + ->getTextContent( + $definition[0] + ) + ]; + } + + // get thumbnail, if it exists + $this->fuckhtml->load($infobox_html[1]); + + $thumb = + $this->fuckhtml + ->getElementsByClassName("float-right", "img"); + + if(count($thumb) !== 0){ + + preg_match( + '/\/image\?img=([^&]+)/i', + $thumb[0]["attributes"]["src"], + $thumb + ); + + if(count($thumb) === 2){ + + $answer["thumb"] = + $this->fuckhtml + ->getTextContent( + $thumb[1] + ); + } + } + + // get description + $ps = + $this->fuckhtml + ->getElementsByTagName("p"); + + $first_tag = true; + foreach($ps as $p){ + + $this->fuckhtml->load($p); + + if( + preg_match( + '/^\s*<strong>/i', + $p["innerHTML"] + ) + ){ + + /* + Parse table + */ + + $strong = + $this->fuckhtml + ->getElementsByTagName("strong")[0]; + + $p["innerHTML"] = + str_replace($strong["innerHTML"], "", $p["innerHTML"]); + + $strong = + preg_replace( + '/:$/', + "", + ucfirst( + $this->fuckhtml + ->getTextContent( + $strong + ) + ) + ); + + $answer["table"][trim($strong)] = + trim( + $this->fuckhtml + ->getTextContent( + $p + ) + ); + + continue; + } + + $as = + $this->fuckhtml + ->getElementsByClassName("svg-icon"); + + if(count($as) !== 0){ + + /* + Parse websites + */ + foreach($as as $a){ + + $answer["sublink"][ + ucfirst(explode(" ", $a["attributes"]["class"], 2)[1]) + ] = + $this->fuckhtml + ->getTextContent( + $a["attributes"]["href"] + ); + } + + continue; + } + + /* + Parse text content + */ + $tags = + $this->fuckhtml + ->getElementsByTagName("*"); + + $i = 0; + foreach($tags as $tag){ + + $c = count($answer["description"]); + + // remove tag from innerHTML + $p["innerHTML"] = + explode($tag["outerHTML"], $p["innerHTML"], 2); + + if(count($p["innerHTML"]) === 2){ + + if( + $i === 0 && + $c !== 0 && + $answer["description"][$c - 1]["type"] == "link" + ){ + + $append = "\n\n"; + }else{ + + $append = ""; + } + + if($p["innerHTML"][0] != ""){ + $answer["description"][] = [ + "type" => "text", + "value" => $append . trim($p["innerHTML"][0]) + ]; + } + + $p["innerHTML"] = $p["innerHTML"][1]; + }else{ + + $p["innerHTML"] = $p["innerHTML"][0]; + } + + switch($tag["tagName"]){ + + case "a": + + $value = + $this->fuckhtml + ->getTextContent( + $tag + ); + + if(strtolower($value) == "wikipedia"){ + + if($c !== 0){ + $answer["description"][$c - 1]["value"] = + rtrim($answer["description"][$c - 1]["value"]); + } + break; + } + + $answer["description"][] = [ + "type" => "link", + "url" => + $this->fuckhtml + ->getTextContent( + $tag["attributes"]["href"] + ), + "value" => + $this->fuckhtml + ->getTextContent( + $tag + ) + ]; + break; + } + + $i++; + } + } + + // get URL + $this->fuckhtml->load($infobox_html[2]); + + $answer["url"] = + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByTagName( + "a" + )[0] + ["attributes"] + ["href"] + ); + + // append answer + $out["answer"][] = $answer; + } + + /* + Get news + */ + $this->fuckhtml->load($html); + + $news = + $this->fuckhtml + ->getElementsByClassName( + "results news-results", + "div" + ); + + if(count($news) !== 0){ + + $this->fuckhtml->load($news[0]); + + $lis = + $this->fuckhtml + ->getElementsByTagName("li"); + + foreach($lis as $li){ + + $this->fuckhtml->load($li); + + $a = + $this->fuckhtml + ->getElementsByClassName( + "ob", + "a" + ); + + if(count($a) === 0){ + + continue; + } + + $a = $a[0]; + + $out["news"][] = [ + "title" => + html_entity_decode( + $this->fuckhtml + ->getTextContent( + $a + ) + ), + "description" => null, + "date" => + strtotime( + explode( + " - ", + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByTagName( + "span" + )[0] + ), + 2 + )[1] + ), + "thumb" => [ + "url" => null, + "ratio" => null + ], + "url" => + $this->fuckhtml + ->getTextContent( + $a["attributes"]["href"] + ) + ]; + } + } + + /* + Get next page + */ + $this->fuckhtml->load($html); + + $pagination = + $this->fuckhtml + ->getElementsByClassName("pagination"); + + if(count($pagination) !== false){ + + $this->fuckhtml->load($pagination[0]); + $as = + $this->fuckhtml + ->getElementsByTagName("a"); + + foreach($as as $a){ + + if($a["innerHTML"] == "Next"){ + + $out["npt"] = $this->nextpage->store( + $this->fuckhtml + ->getTextContent( + $a["attributes"]["href"] + ), + "web" + ); + } + } + } + + return $out; + } + + public function news($get){ + + $search = $get["s"]; + + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $out = [ + "status" => "ok", + "npt" => null, + "news" => [] + ]; + + try{ + $html = + $this->get( + "https://www.mojeek.com/search", + [ + "q" => $search, + "fmt" => "news" + ] + ); + }catch(Exception $error){ + + throw new Exception("Failed to get HTML"); + } + + /* + $handle = fopen("scraper/mojeek.html", "r"); + $html = fread($handle, filesize("scraper/mojeek.html")); + fclose($handle);*/ + + /* + Get big, standard and smaller nodes + */ + foreach( + [ + "results-extended", + "results-standard" + ] + as $categoryname + ){ + + $this->fuckhtml->load($html); + + $categories = + $this->fuckhtml + ->getElementsByClassName( + $categoryname, + "ul" + ); + + foreach($categories as $category){ + + $this->fuckhtml->load($category); + + $nodes = + $this->fuckhtml + ->getElementsByTagName("li"); + + foreach($nodes as $node){ + + $data = [ + "title" => null, + "author" => null, + "description" => null, + "date" => null, + "thumb" => + [ + "url" => null, + "ratio" => null + ], + "url" => null + ]; + + /* + Parse the results + */ + $this->fuckhtml->load($node); + + // get title + url + $a = + $this->fuckhtml + ->getElementsByTagName("a")[0]; + + $data["title"] = + $this->fuckhtml + ->getTextContent( + $a["attributes"]["title"] + ); + + $data["url"] = + $this->fuckhtml + ->getTextContent( + $a["attributes"]["href"] + ); + + // get image + $image = + $this->fuckhtml + ->getElementsByTagName("img"); + + if(count($image) !== 0){ + + $data["thumb"] = [ + "url" => + urldecode( + str_replace( + "/image?img=", + "", + $this->fuckhtml + ->getTextContent( + $image[0]["attributes"]["src"] + ) + ) + ), + "ratio" => "16:9" + ]; + } + + // get description + $description = + $this->fuckhtml + ->getElementsByClassName("s", "p"); + + if(count($description) !== 0){ + + $data["description"] = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $description[0] + ) + ); + } + + // get date + time + $date = + $this->fuckhtml + ->getElementsByClassName( + "date", + "p" + ); + + $i = + $this->fuckhtml + ->getElementsByClassName("i", "p"); + + if(count($date) !== 0){ + + // we're inside a big node + $data["date"] = strtotime($date[0]["innerHTML"]); + + if(count($i) !== 0){ + + $this->fuckhtml->load($i[0]); + + $a = + $this->fuckhtml + ->getElementsByTagName("a"); + + if(count($a) !== 0){ + + $data["author"] = + $this->fuckhtml + ->getTextContent($a[0]); + } + } + }else{ + + // we're inside a small node + if(count($i) !== 0){ + + $i = + explode( + " - ", + $this->fuckhtml + ->getTextContent($i[0]) + ); + + $data["date"] = strtotime(array_pop($i)); + $data["author"] = implode(" - ", $i); + } + } + + $out["news"][] = $data; + } + } + } + + return $out; + } + + private function titledots($title){ + + return trim($title, ". \t\n\r\0\x0B"); + } +} + diff --git a/scraper/wiby.php b/scraper/wiby.php new file mode 100644 index 0000000..a1daf57 --- /dev/null +++ b/scraper/wiby.php @@ -0,0 +1,244 @@ +<?php + +class wiby{ + + public function __construct(){ + + include "lib/nextpage.php"; + $this->nextpage = new nextpage("wiby"); + } + + public function getfilters($page){ + + if($page != "web"){ + + return []; + } + + return [ + "nsfw" => [ + "display" => "NSFW", + "option" => [ + "yes" => "Yes", + "no" => "No" + ] + ], + "date" => [ + "display" => "Time posted", + "option" => [ + "any" => "Any time", + "day" => "Past day", + "week" => "Past week", + "month" => "Past month", + "year" => "Past year", + ] + ] + ]; + } + + private function get($url, $get = [], $nsfw){ + + $curlproc = curl_init(); + + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + + curl_setopt($curlproc, CURLOPT_URL, $url); + + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curlproc, CURLOPT_HTTPHEADER, + ["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip", + "Cookie: ws={$nsfw}", + "DNT: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1"] + ); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + public function web($get){ + + if($get["npt"]){ + + $q = + json_decode( + $this->nextpage->get($get["npt"], "web"), + true + ); + + $nsfw = $q["nsfw"]; + unset($q["nsfw"]); + }else{ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $date = $get["date"]; + $nsfw = $get["nsfw"] == "yes" ? "0" : "1"; + + $search = + str_replace( + [ + "!g", + "!gi", + "!gv", + "!gm", + "!b", + "!bi", + "!bv", + "!bm", + "!td", + "!tw", + "!tm", + "!ty", + "&g", + "&gi", + "&gv", + "&gm", + "&b", + "&bi", + "&bv", + "&bm", + "&td", + "&tw", + "&tm", + "&ty", + ], + "", + $search + ); + + switch($date){ + + case "day": $search = "!td " . $search; break; + case "week": $search = "!tw " . $search; break; + case "month": $search = "!tm " . $search; break; + case "year": $search = "!ty " . $search; break; + } + + $q = [ + "q" => $search + ]; + } + + try{ + $html = $this->get( + "https://wiby.me/", + $q, + $nsfw + ); + }catch(Exception $error){ + + throw new Exception("Failed to fetch search page"); + } + + preg_match( + '/<p class="pin"><blockquote>(?:<\/p>)?<br><a class="more" href="\/\?q=[^"]+&p=([0-9]+)">Find more\.\.\.<\/a><\/blockquote>/', + $html, + $nextpage + ); + + if(count($nextpage) === 0){ + + $nextpage = null; + }else{ + + $nextpage = + $this->nextpage->store( + json_encode([ + "q" => $q["q"], + "p" => (int)$nextpage[1], + "nsfw" => $nsfw + ]), + "web" + ); + } + + $out = [ + "status" => "ok", + "spelling" => [ + "type" => "no_correction", + "using" => null, + "correction" => null + ], + "npt" => $nextpage, + "answer" => [], + "web" => [], + "image" => [], + "video" => [], + "news" => [], + "related" => [] + ]; + + preg_match_all( + '/<blockquote>[\s]*<a .* href="(.*)">(.*)<\/a>.*<p>(.*)<\/p>[\s]*<\/blockquote>/Ui', + $html, + $links + ); + + for($i=0; $i<count($links[0]); $i++){ + + $out["web"][] = [ + "title" => $this->unescapehtml(trim($links[2][$i])), + "description" => $this->unescapehtml(trim(strip_tags($links[3][$i]))), + "url" => trim($links[1][$i]), + "date" => null, + "type" => "web", + "thumb" => [ + "url" => null, + "ratio" => null + ], + "sublink" => [], + "table" => [] + ]; + } + + return $out; + } + + private function unescapehtml($str){ + + return html_entity_decode( + str_replace( + [ + "<br>", + "<br/>", + "</br>", + "<BR>", + "<BR/>", + "</BR>", + ], + "\n", + $str + ), + ENT_QUOTES | ENT_XML1, 'UTF-8' + ); + } +} diff --git a/scraper/yandex.php b/scraper/yandex.php new file mode 100644 index 0000000..437c8aa --- /dev/null +++ b/scraper/yandex.php @@ -0,0 +1,530 @@ +<?php + +class yandex{ + + /* + curl functions + */ + public function __construct(){ + + include "lib/fuckhtml.php"; + $this->fuckhtml = new fuckhtml(); + + include "lib/nextpage.php"; + $this->nextpage = new nextpage("yandex"); + } + + private function get($url, $get = [], $nsfw){ + + $curlproc = curl_init(); + + $search = $get["text"]; + + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + + curl_setopt($curlproc, CURLOPT_URL, $url); + + switch($nsfw){ + case "yes": $nsfw = "0"; break; + case "maybe": $nsfw = "1"; break; + case "no": $nsfw = "2"; break; + } + + $headers = + ["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Encoding: gzip", + "Accept-Language: en-US,en;q=0.5", + "DNT: 1", + "Cookie: yp=1716337604.sp.family%3A{$nsfw}#1685406411.szm.1:1920x1080:1920x999", + "Referer: https://yandex.com/images/search?text={$search}", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: cross-site", + "Upgrade-Insecure-Requests: 1"]; + + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + public function getfilters($pagetype){ + + switch($pagetype){ + + case "images": + return + [ + "nsfw" => [ + "display" => "NSFW", + "option" => [ + "yes" => "Yes", + "maybe" => "Maybe", + "no" => "No" + ] + ], + "time" => [ + "display" => "Time posted", + "option" => [ + "any" => "Any time", + "week" => "Last week" + ] + ], + "size" => [ + "display" => "Size", + "option" => [ + "any" => "Any size", + "small" => "Small", + "medium" => "Medium", + "large" => "Large", + "wallpaper" => "Wallpaper" + ] + ], + "color" => [ + "display" => "Colors", + "option" => [ + "any" => "All colors", + "color" => "Color images only", + "gray" => "Black and white", + "red" => "Red", + "orange" => "Orange", + "yellow" => "Yellow", + "cyan" => "Cyan", + "green" => "Green", + "blue" => "Blue", + "violet" => "Purple", + "white" => "White", + "black" => "Black" + ] + ], + "type" => [ + "display" => "Type", + "option" => [ + "any" => "All types", + "photo" => "Photos", + "clipart" => "White background", + "lineart" => "Drawings and sketches", + "face" => "People", + "demotivator" => "Demotivators" + ] + ], + "layout" => [ + "display" => "Layout", + "option" => [ + "any" => "All layouts", + "horizontal" => "Horizontal", + "vertical" => "Vertical", + "square" => "Square" + ] + ], + "format" => [ + "display" => "Format", + "option" => [ + "any" => "Any format", + "jpeg" => "JPEG", + "png" => "PNG", + "gif" => "GIF" + ] + ] + ]; + break; + + default: + return []; + break; + } + } + + public function image($get){ + + if($get["npt"]){ + + $request = + json_decode( + $this->nextpage->get( + $get["npt"], + "images" + ), + true + ); + + $nsfw = $request["nsfw"]; + unset($request["nsfw"]); + }else{ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $nsfw = $get["nsfw"]; + $time = $get["time"]; + $size = $get["size"]; + $color = $get["color"]; + $type = $get["type"]; + $layout = $get["layout"]; + $format = $get["format"]; + /* + $handle = fopen("scraper/yandex.json", "r"); + $json = fread($handle, filesize("scraper/yandex.json")); + fclose($handle);*/ + + // SIZE + // large + // 227.0=1;203.0=1;76fe94.0=1;41d251.0=1;75.0=1;371.0=1;291.0=1;307.0=1;f797ee.0=1;1cf7c2.0=1;deca32.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&isize=large&suggest_reqid=486139416166165501540886508227485&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // medium + // 227.0=1;203.0=1;76fe94.0=1;41d251.0=1;75.0=1;371.0=1;291.0=1;307.0=1;f797ee.0=1;1cf7c2.0=1;deca32.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&isize=medium&suggest_reqid=486139416166165501540886508227485&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // small + // 227.0=1;203.0=1;76fe94.0=1;41d251.0=1;75.0=1;371.0=1;291.0=1;307.0=1;f797ee.0=1;1cf7c2.0=1;deca32.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&isize=small&suggest_reqid=486139416166165501540886508227485&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // ORIENTATION + // Horizontal + // 227.0=1;203.0=1;76fe94.0=1;41d251.0=1;75.0=1;371.0=1;291.0=1;307.0=1;f797ee.0=1;1cf7c2.0=1;deca32.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&iorient=horizontal&suggest_reqid=486139416166165501540886508227485&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // Vertical + // 227.0=1;203.0=1;76fe94.0=1;41d251.0=1;75.0=1;371.0=1;291.0=1;307.0=1;f797ee.0=1;1cf7c2.0=1;deca32.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&iorient=vertical&suggest_reqid=486139416166165501540886508227485&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // Square + // 227.0=1;203.0=1;76fe94.0=1;41d251.0=1;75.0=1;371.0=1;291.0=1;307.0=1;f797ee.0=1;1cf7c2.0=1;deca32.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&iorient=square&suggest_reqid=486139416166165501540886508227485&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // TYPE + // Photos + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&text=minecraft&type=photo&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // White background + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&text=minecraft&type=clipart&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // Drawings and sketches + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&text=minecraft&type=lineart&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // People + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&text=minecraft&type=face&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // Demotivators + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&text=minecraft&type=demotivator&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // COLOR + // Color images only + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&icolor=color&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // Black and white + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&icolor=gray&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // Red + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&icolor=red&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // Orange + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&icolor=orange&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // Yellow + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&icolor=yellow&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // Cyan + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&icolor=cyan&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // Green + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&icolor=green&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // Blue + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&icolor=blue&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // Purple + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&icolor=violet&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // White + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&icolor=white&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // Black + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&icolor=black&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // FORMAT + // jpeg + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&itype=jpg&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // png + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&itype=png&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // gif + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&itype=gifan&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // RECENT + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&recent=7D&text=minecraft&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + // WALLPAPER + // 307.0=1;371.0=1;291.0=1;203.0=1;deca32.0=1;f797ee.0=1;1cf7c2.0=1;41d251.0=1;267.0=1;bde197.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}&yu=4861394161661655015&isize=wallpaper&text=minecraft&wp=wh16x9_1920x1080&uinfo=sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080 + + + $request = [ + "format" => "json", + "request" => [ + "blocks" => [ + [ + "block" => "extra-content", + "params" => (object)[], + "version" => 2 + ], + [ + "block" => "i-global__params:ajax", + "params" => (object)[], + "version" => 2 + ], + [ + "block" => "search2:ajax", + "params" => (object)[], + "version" => 2 + ], + [ + "block" => "preview__isWallpaper", + "params" => (object)[], + "version" => 2 + ], + [ + "block" => "content_type_search", + "params" => (object)[], + "version" => 2 + ], + [ + "block" => "serp-controller", + "params" => (object)[], + "version" => 2 + ], + [ + "block" => "cookies_ajax", + "params" => (object)[], + "version" => 2 + ], + [ + "block" => "advanced-search-block", + "params" => (object)[], + "version" => 2 + ] + ], + "metadata" => [ + "bundles" => [ + "lb" => "AS?(E<X120" + ], + "assets" => [ + // las base + "las" => "justifier-height=1;justifier-setheight=1;fitimages-height=1;justifier-fitincuts=1;react-with-dom=1;" + + // las default + //"las" => "justifier-height=1;justifier-setheight=1;fitimages-height=1;justifier-fitincuts=1;react-with-dom=1;227.0=1;203.0=1;76fe94.0=1;215f96.0=1;75.0=1" + ], + "extraContent" => [ + "names" => [ + "i-react-ajax-adapter" + ] + ] + ] + ] + ]; + + /* + Apply filters + */ + if($time == "week"){ + $request["recent"] = "7D"; + } + + if($size != "any"){ + + $request["isize"] = $size; + } + + if($type != "any"){ + + $request["type"] = $type; + } + + if($color != "any"){ + + $request["icolor"] = $color; + } + + if($layout != "any"){ + + $request["iorient"] = $layout; + } + + if($format != "any"){ + + $request["itype"] = $format; + } + + $request["text"] = $search; + $request["uinfo"] = "sw-1920-sh-1080-ww-1125-wh-999-pd-1-wp-16x9_1920x1080"; + + $request["request"] = json_encode($request["request"]); + } + + try{ + $json = $this->get( + "https://yandex.com/images/search", + $request, + $nsfw + ); + }catch(Exception $err){ + + throw new Exception("Failed to get JSON"); + } + /* + $handle = fopen("scraper/yandex.json", "r"); + $json = fread($handle, filesize("scraper/yandex.json")); + fclose($handle);*/ + + $json = json_decode($json, true); + + if( + isset($json["type"]) && + $json["type"] == "captcha" + ){ + + throw new Exception("Yandex blocked this 4get instance. Yandex blocks don't last very long, but the block timer gets reset everytime you make another unsuccessful request. Please try again in ~7 minutes."); + } + + if($json === null){ + + throw new Exception("Failed to decode JSON"); + } + + // get html + $html = ""; + foreach($json["blocks"] as $block){ + + $html .= $block["html"]; + } + + $this->fuckhtml->load($html); + $div = $this->fuckhtml->getElementsByTagName("div"); + + $out = [ + "status" => "ok", + "npt" => null, + "image" => [] + ]; + + // check for next page + if( + count( + $this->fuckhtml + ->getElementsByClassName( + "more more_direction_next", + $div + ) + ) !== 0 + ){ + + $request["nsfw"] = $nsfw; + + if(isset($request["p"])){ + + $request["p"]++; + }else{ + + $request["p"] = 1; + } + + $out["npt"] = $this->nextpage->store(json_encode($request), "images"); + } + + // get search results + foreach( + $this->fuckhtml + ->getElementsByClassName( + "serp-item serp-item_type_search", + $div + ) + as $image + ){ + + $image = + json_decode( + $image + ["attributes"] + ["data-bem"], + true + )["serp-item"]; + + $title = [html_entity_decode($image["snippet"]["title"], ENT_QUOTES | ENT_HTML5)]; + + if(isset($image["snippet"]["text"])){ + + $title[] = html_entity_decode($image["snippet"]["text"], ENT_QUOTES | ENT_HTML5); + } + + $tmp = [ + "title" => + $this->fuckhtml + ->getTextContent( + $this->titledots( + implode(": ", $title) + ) + ), + "source" => [], + "url" => htmlspecialchars_decode($image["snippet"]["url"]) + ]; + + foreach($image["dups"] as $dup){ + + $tmp["source"][] = [ + "url" => htmlspecialchars_decode($dup["url"]), + "width" => (int)$dup["w"], + "height" => (int)$dup["h"], + ]; + } + + $tmp["source"][] = [ + "url" => + preg_replace( + '/^\/\//', + "https://", + htmlspecialchars_decode($image["thumb"]["url"]) + ), + "width" => (int)$image["thumb"]["size"]["width"], + "height" => (int)$image["thumb"]["size"]["height"] + ]; + + $out["image"][] = $tmp; + } + + return $out; + } + + private function titledots($title){ + + $substr = substr($title, -3); + + if( + $substr == "..." || + $substr == "…" + ){ + + return trim(substr($title, 0, -3)); + } + + return trim($title); + } +} diff --git a/scraper/youtube.php b/scraper/youtube.php new file mode 100644 index 0000000..83a68ba --- /dev/null +++ b/scraper/youtube.php @@ -0,0 +1,1723 @@ +<?php + +//$yt = new youtube(); +//header("Content-Type: application/json"); +//echo json_encode($yt->video("minecraft", null, "today", "any", "any", "live", "relevance")); + +class youtube{ + + public function __construct(){ + + include "lib/nextpage.php"; + $this->nextpage = new nextpage("yt"); + } + + public function getfilters($page){ + + if($page != "videos"){ + + return []; + } + + return [ + "date" => [ + "display" => "Time posted", + "option" => [ + "any" => "Any time", + "hour" => "Last hour", + "today" => "Today", + "week" => "This week", + "month" => "This month", + "year" => "This year" + ] + ], + "type" => [ + "display" => "Type", + "option" => [ + "video" => "Video", + "channel" => "Channel", + "playlist" => "Playlist", + "Movie" => "Movie" + ] + ], + "duration" => [ + "display" => "Duration", + "option" => [ + "any" => "Any duration", + "short" => "Short (>4min)", + "medium" => "Medium (4-20min)", + "long" => "Long (<20min)" + ] + ], + "feature" => [ + "display" => "Feature", + "option" => [ + "any" => "No features", + "live" => "Live", + "4k" => "4K", + "hd" => "HD", + "subtitles" => "Subtitles/CC", + "creativecommons" => "Creative Commons", + "360" => "VR 360°", + "vr180" => "VR 180°", + "3d" => "3D", + "hdr" => "HDR" + ] + ], + "sort" => [ + "display" => "Sort by", + "option" => [ + "relevance" => "Relevance", + "upload_date" => "Upload date", + "view_count" => "View count", + "rating" => "Rating" + ] + ] + ]; + } + + private function ytfilter($date, $type, $duration, $feature, $sort){ + + // ------------ + // INCOMPATIBLE FILTERS + // channel,playlist DURATION, FEATURES, SORT BY + // Movie Features=[live, subtitles, creative commons, 3d] + + // live, 3D + // Type[channel, playlist, movie] + + // UPLOAD DATE, DURATION, 4k, 360, VR180, HDR + // Type[channel, playlist] + + // ----------- + + // MUST BE TOGETHER + // Relevance,upload date Type=Video + + switch($type){ + + case "channel": + case "playlist": + if($duration != "any"){ $duration = "any"; } + if($feature != "any"){ $feature = "any"; } + if($sort != "any"){ $sort = "any"; } + break; + + case "movie": + if( + in_array( + $feature, + [ + "live", + "subtitles", + "creative_commons", + "3d" + ], + ) + ){ + + $feature = "any"; + } + break; + } + + switch($feature){ + + case "live": + case "3d": + if( + in_array( + $type, + [ + "channel", + "playlist", + "movie" + ], + ) + ){ + + $type = "video"; + } + break; + } + + if( + ( + $date != "any" || + $duration != "any" || + $feature == "4k" || + $feature == "360" || + $feature == "vr180" || + $feature == "hdr" + ) && + ( + $type == "channel" || + $type == "playlist" + ) + ){ + + $type = "video"; + } + + if( + $date == "any" && + $type == "video" && + $duration == "any" && + $feature == "any" && + $sort == "relevance" + ){ + + return null; + } + + //print_r([$date, $type, $duration, $feature, $sort]); + + /* + Encode hex data + */ + + // UPLOAD DATE + // hour EgQIARAB 12 04 08 01 10 01 + // today EgQIAhAB 12 04 08 02 10 01 + // week EgQIAxAB 12 04 08 03 10 01 + // month EgQIBBAB 12 04 08 04 10 01 + // year EgQIBRAB 12 04 08 05 10 01 + + // TYPE + // video EgIQAQ%253D%253D 12 02 10 01 + // channel EgIQAg%253D%253D 12 02 10 02 + // playlist EgIQAw%253D%253D 12 02 10 03 + // movie EgIQBA%253D%253D 12 02 10 04 + + // DURATION + // -4min EgIYAQ%253D%253D 12 02 18 01 + // 4-20min EgIYAw%253D%253D 12 02 18 03 + // 20+min EgIYAg%253D%253D 12 02 18 02 + + // FEATURE + // live EgJAAQ%253D%253D 12 02 40 01 + // 4K EgJwAQ%253D%253D 12 02 70 01 + // HD EgIgAQ%253D%253D 12 02 20 01 + // Subtitles/CC EgIoAQ%253D%253D 12 02 28 01 + // Creative Commons EgIwAQ%253D%253D 12 02 30 01 + // 360 EgJ4AQ%253D%253D 12 02 78 01 + // VR180 EgPQAQE%253D 12 03 d0 01 01 + // 3D EgI4AQ%253D%253D 12 02 38 01 + // HDR EgPIAQE%253D 12 03 c8 01 01 + // (location & purchased unused) + + // SORT BY + // Relevance CAASAhAB 08 00 12 02 10 01 (is nothing by default) + // Upload date CAI%253D 08 02 + // View count CAM%253D 08 03 + // Rating CAE%253D 08 01 + + // video + // 12 02 10 01 + + // under 4 minutes + // 12 02 18 01 + + // video + under 4 minutes + // 12 04 10 01 18 01 + + // video + under 4 minutes + HD + // 08 00 12 06 10 01 18 01 20 01 + + // video + under 4 minutes + upload date + // 08 02 12 04 10 01 18 01 + + // video + under 4 minutes + HD + upload date + // 08 02 12 06 10 01 18 01 20 01 + + // this year + video + under 4 minutes + HD + upload date + // 08 02 12 08 08 05 10 01 18 01 20 01 + + // this week + video + over 20 minutes + HD + view count + // 08 03 12 08 08 03 10 01 18 02 20 01 + + //echo urlencode(urlencode(base64_encode(hex2bin($str)))); + //echo bin2hex(base64_decode(urldecode(urldecode("CAI%253D")))); + + // week + video + 20min + rating + // 08 01 12 06 08 03 10 01 18 02 + + // week + video + 20min + live + rating + // 08 01 12 08 08 03 10 01 18 02 40 01 + + // live 12 02 40 01 + + $hex = null; + if( + $date == "any" && + $type == "video" && + $duration == "any" && + $feature == "any" && + $sort == "relevance" + ){ + + return $hex; + } + + $opcode = 0; + + if($date != "any"){ $opcode += 2; } + if($type != "any"){ $opcode += 2; } + if($duration != "any"){ $opcode += 2; } + + switch($feature){ + + case "live": + case "4k": + case "hd": + case "subtitles": + case "creativecommons": + case "360": + case "3d": + $opcode += 2; + break; + + case "hdr": + case "vr180": + $opcode += 3; + break; + } + + switch($sort){ + + case "relevance": $hex .= "0800"; break; + case "upload_date": $hex .= "0802"; break; + case "view_count": $hex .= "0803"; break; + case "rating": $hex .= "0801"; break; + } + + $hex .= "12" . "0".$opcode; + + switch($date){ + + case "hour": $hex .= "0801"; break; + case "today": $hex .= "0802"; break; + case "week": $hex .= "0803"; break; + case "month": $hex .= "0804"; break; + case "year": $hex .= "0805"; break; + } + + switch($type){ + + case "video": $hex .= "1001"; break; + case "channel": $hex .= "1002"; break; + case "playlist": $hex .= "1003"; break; + case "movie": $hex .= "1004"; break; + } + + switch($duration){ + + case "short": $hex .= "1801"; break; + case "medium": $hex .= "1803"; break; + case "long": $hex .= "1802"; break; + } + + switch($feature){ + + case "live": $hex .= "4001"; break; + case "4k": $hex .= "7001"; break; + case "hd": $hex .= "2001"; break; + case "subtitles": $hex .= "2801"; break; + case "creativecommons": $hex .= "3001"; break; + case "360": $hex .= "7801"; break; + case "vr180": $hex .= "d00101"; break; + case "3d": $hex .= "3801"; break; + case "hdr": $hex .= "c80101"; break; + } + + //echo $hex . "\n\n"; + return urlencode(base64_encode(hex2bin($hex))); + } + + // me reading youtube's json + // https://imgur.com/X9hVlFX + + const req_web = 0; + const req_xhr = 1; + + private function get($url, $get = [], $reqtype = self::req_web, $continuation = null){ + + $curlproc = curl_init(); + + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + + curl_setopt($curlproc, CURLOPT_URL, $url); + + switch($reqtype){ + case self::req_web: + $headers = + ["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip", + "Cookie: PREF=tz=America.New_York", + "DNT: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1"]; + break; + + case self::req_xhr: + $headers = + ["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:110.0) Gecko/20100101 Firefox/110.0", + "Accept: */*", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip", + "Cookie: PREF=tz=America.New_York", + "Referer: https://youtube.com.com/", + "Content-Type: application/json", + "Content-Length: " . strlen($continuation), + "DNT: 1", + "Connection: keep-alive", + "Sec-Fetch-Dest: empty", + "Sec-Fetch-Mode: same-origin", + "Sec-Fetch-Site: same-origin"]; + + curl_setopt($curlproc, CURLOPT_POST, true); + curl_setopt($curlproc, CURLOPT_POSTFIELDS, $continuation); + break; + } + + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + public function video($get){ + + $this->out = [ + "status" => "ok", + "npt" => null, + "video" => [], + "author" => [], + "livestream" => [], + "playlist" => [], + "reel" => [] + ]; + + if($get["npt"]){ + + // parse nextPage + // https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false + /* + $handle = fopen("nextpage.json", "r"); + $json = fread($handle, filesize("nextpage.json")); + fclose($handle);*/ + + $npt = + json_decode( + $this->nextpage->get( + $get["npt"], + "videos" + ), + true + ); + + try{ + $json = $this->get( + "https://www.youtube.com/youtubei/v1/search", + [ + "key" => $npt["key"], + "prettyPrint" => "false" + ], + self::req_xhr, + json_encode($npt["post"]) + ); + }catch(Exception $error){ + + throw new Exception("Could not fetch results page"); + } + + $json = json_decode($json); + + foreach( + $json + ->onResponseReceivedCommands[0] + ->appendContinuationItemsAction + ->continuationItems[0] + ->itemSectionRenderer + ->contents + as $video + ){ + + $this->parsevideoobject($video); + } + + if( + !isset( + $json + ->onResponseReceivedCommands[0] + ->appendContinuationItemsAction + ->continuationItems[1] + ->continuationItemRenderer + ->continuationEndpoint + ->continuationCommand + ->token + ) + ){ + + $npt = null; + + }else{ + // prepare nextpage for later.. + $npt["post"]["continuation"] = + $json + ->onResponseReceivedCommands[0] + ->appendContinuationItemsAction + ->continuationItems[1] + ->continuationItemRenderer + ->continuationEndpoint + ->continuationCommand + ->token; + } + + $this->out["npt"] = $npt; + + }else{ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $date = $get["date"]; + $type = $get["type"]; + $duration = $get["duration"]; + $feature = $get["feature"]; + $sort = $get["sort"]; + + // parse ytInitialData + + $get = [ + "search_query" => $search + ]; + + if( + ( + $filter = + $this->ytfilter( + $date, + $type, + $duration, + $feature, + $sort + ) + ) !== null + ){ + + $get["sp"] = $filter; + } + + try{ + $json = $this->get( + "https://www.youtube.com/results", + $get + ); + }catch(Exception $error){ + + throw new Exception("Could not fetch results page"); + } + /* + $handle = fopen("test.html", "r"); + $json = fread($handle, filesize("test.html")); + fclose($handle); + */ + if( + !preg_match( + '/ytcfg\.set\(({".*})\); *window\.ytcfg/', + $json, + $ytconfig + ) + ){ + + throw new Exception("Could not get ytcfg"); + } + + $ytconfig = json_decode($ytconfig[1]); + + if( + !preg_match( + '/ytInitialData *= *({.*});<\/script>/', + $json, + $json + ) + ){ + + throw new Exception("Could not get ytInitialData"); + } + + $json = json_decode($json[1]); + + // generate POST data for nextpage + + $ytconfig->INNERTUBE_CONTEXT->client->screenWidthPoints = 1239; + $ytconfig->INNERTUBE_CONTEXT->client->screenHeightPoints = 999; + $ytconfig->INNERTUBE_CONTEXT->client->screenPixelDensity = 1; + $ytconfig->INNERTUBE_CONTEXT->client->screenDensityFloat = 1; + $ytconfig->INNERTUBE_CONTEXT->client->utcOffsetMinutes = -240; + $ytconfig->INNERTUBE_CONTEXT->request->internalExperimentFlags = []; + $ytconfig->INNERTUBE_CONTEXT->request->consistencyTokenJars = []; + + $ytconfig->INNERTUBE_CONTEXT->client->mainAppWebInfo = [ + "graftUrl" => $ytconfig->INNERTUBE_CONTEXT->client->originalUrl, + "webDisplayMode" => "WEB_DISPLAY_MODE_BROWSER", + "isWebNativeShareAvailable" => false + ]; + + $ytconfig->INNERTUBE_CONTEXT->adSignalsInfo = [ + "params" => [ + [ + "key" => "dt", + "value" => (string)$ytconfig->TIME_CREATED_MS + ], + [ + "key" => "flash", + "value" => "0" + ], + [ + "key" => "frm", + "value" => "0" + ], + [ + "key" => "u_tz", + "value" => "-240" + ], + [ + "key" => "u_his", + "value" => "3" + ], + [ + "key" => "u_h", + "value" => "1080" + ], + [ + "key" => "u_w", + "value" => "1920" + ], + [ + "key" => "u_ah", + "value" => "1080" + ], + [ + "key" => "u_cd", + "value" => "24" + ], + [ + "key" => "bc", + "value" => "31" + ], + [ + "key" => "bih", + "value" => "999" + ], + [ + "key" => "biw", + "value" => "1239" + ], + [ + "key" => "brdim", + "value" => "0,0,0,0,1920,0,1920,1061,1239,999" + ], + [ + "key" => "vis", + "value" => "1" + ], + [ + "key" => "wgl", + "value" => "true" + ], + [ + "key" => "ca_type", + "value" => "image" + ] + ] + ]; + + /* + echo json_encode($json); + die();*/ + + // *inhales* + foreach( + $json + ->contents + ->twoColumnSearchResultsRenderer + ->primaryContents + ->sectionListRenderer + ->contents[0] + ->itemSectionRenderer + ->contents + as $video + ){ + + $this->parsevideoobject($video); + } + + // get additional data from secondaryContents + if( + isset( + $json + ->contents + ->twoColumnSearchResultsRenderer + ->secondaryContents + ->secondarySearchContainerRenderer + ->contents[0] + ->universalWatchCardRenderer + ) + ){ + + $video = + $json + ->contents + ->twoColumnSearchResultsRenderer + ->secondaryContents + ->secondarySearchContainerRenderer + ->contents[0] + ->universalWatchCardRenderer; + /* + echo json_encode($video); + die();*/ + + $author = + [ + "name" => + $video + ->header + ->watchCardRichHeaderRenderer + ->title + ->simpleText, + "url" => + "https://www.youtube.com/channel/" . + $video + ->header + ->watchCardRichHeaderRenderer + ->titleNavigationEndpoint + ->browseEndpoint + ->browseId, + "avatar" => null + ]; + + if( + isset( + $video + ->header + ->watchCardRichHeaderRenderer + ->avatar + ->thumbnails[0] + ->url + ) + ){ + + $author["avatar"] = + $video + ->header + ->watchCardRichHeaderRenderer + ->avatar + ->thumbnails[0] + ->url; + } + + // add video in callToAction if present + if( + isset( + $video + ->callToAction + ->watchCardHeroVideoRenderer + ->lengthText + ) + ){ + + array_push( + $this->out["video"], + [ + "title" => + $video + ->callToAction + ->watchCardHeroVideoRenderer + ->title + ->simpleText, + "description" => null, + "author" => $author, + "date" => + $this->textualdate2unix( + trim( + explode( + "•", + $video + ->callToAction + ->watchCardHeroVideoRenderer + ->subtitle + ->simpleText + )[2] + ) + ), + "duration" => + $this->hms2int( + $video + ->callToAction + ->watchCardHeroVideoRenderer + ->lengthText + ->simpleText + ), + "views" => + $this->truncatedcount2int( + trim( + explode( + "•", + $video + ->callToAction + ->watchCardHeroVideoRenderer + ->subtitle + ->simpleText, + 2 + )[1] + ) + ), + "thumb" => [ + "url" => + $video + ->callToAction + ->watchCardHeroVideoRenderer + ->heroImage + ->singleHeroImageRenderer + ->thumbnail + ->thumbnails[0] + ->url, + "ratio" => "16:9" + ], + "url" => + "https://www.youtube.com/watch?v=" . + $video + ->callToAction + ->watchCardHeroVideoRenderer + ->navigationEndpoint + ->watchEndpoint + ->videoId + ] + ); + } + + // get all playlists, ignore videos + $out = null; + + foreach( + $video + ->sections + as $section + ){ + + if( + isset( + $section + ->watchCardSectionSequenceRenderer + ->lists[0] + ->horizontalCardListRenderer + ->cards + ) + ){ + + $out = + $section + ->watchCardSectionSequenceRenderer + ->lists[0] + ->horizontalCardListRenderer + ->cards; + break; + } + } + + if($out !== null){ + + foreach( + $out as $video + ){ + + if( + !isset( + $video + ->searchRefinementCardRenderer + ) + ){ + + continue; + } + + $video = + $video + ->searchRefinementCardRenderer; + + array_push( + $this->out["playlist"], + [ + "title" => + $video + ->query + ->runs[0] + ->text, + "description" => null, + "author" => $author, + "date" => null, + "duration" => null, + "views" => null, + "thumb" => [ + "url" => + $video + ->thumbnail + ->thumbnails[0] + ->url, + "ratio" => "1:1" + ], + "url" => + "https://www.youtube.com" . + $video + ->searchEndpoint + ->commandMetadata + ->webCommandMetadata + ->url + ] + ); + } + } + } + + foreach( + $json + ->contents + ->twoColumnSearchResultsRenderer + ->primaryContents + ->sectionListRenderer + ->contents + as $cont + ){ + + if(isset($cont->continuationItemRenderer)){ + + $this->out["npt"] = [ + "key" => + $ytconfig + ->INNERTUBE_API_KEY, + "post" => [ + "context" => + $ytconfig + ->INNERTUBE_CONTEXT, + "continuation" => + $cont + ->continuationItemRenderer + ->continuationEndpoint + ->continuationCommand + ->token + ] + ]; + break; + } + } + } + + if($this->out["npt"] !== null){ + + $this->out["npt"] = $this->nextpage->store(json_encode($this->out["npt"]), "videos"); + } + + return $this->out; + } + + private function parsevideoobject($video){ + + if(isset($video->videoRenderer)){ + + $video = $video->videoRenderer; + + $description = null; + + if(isset($video->detailedMetadataSnippets)){ + foreach( + $video + ->detailedMetadataSnippets[0] + ->snippetText + ->runs + as $description_part + ){ + + $description .= $description_part->text; + } + } + + if( + isset( + $video + ->badges[0] + ->metadataBadgeRenderer + ->icon + ->iconType + ) && + $video + ->badges[0] + ->metadataBadgeRenderer + ->icon + ->iconType + == "LIVE" + ){ + + $type = "livestream"; + $date = null; + $duration = "_LIVE"; + + if(isset($video->viewCountText->runs[0]->text)){ + + $views = + $this->views2int( + $video + ->viewCountText + ->runs[0] + ->text + ); + }else{ + + $views = null; + } + }else{ + + $type = "video"; + + if(isset($video->publishedTimeText->simpleText)){ + + $date = $this->textualdate2unix( + $video + ->publishedTimeText + ->simpleText + ); + }else{ + + $date = null; + } + + if(isset($video->lengthText->simpleText)){ + + $duration = + $this->hms2int( + $video + ->lengthText + ->simpleText + ); + }else{ + + $duration = null; + } + + if(isset($video->viewCountText->simpleText)){ + + $views = + $this->views2int( + $video + ->viewCountText + ->simpleText + ); + }else{ + + $views = null; + } + } + + if( + $video + ->navigationEndpoint + ->commandMetadata + ->webCommandMetadata + ->webPageType + == "WEB_PAGE_TYPE_SHORTS" + ){ + + // haha you thought you could get me, youtube + // jokes on you i dont go outside + $type = "reel"; + } + + array_push( + $this->out[$type], + [ + "title" => + $video + ->title + ->runs[0] + ->text, + "description" => + $this->titledots($description), + "author" => [ + "name" => + $video + ->longBylineText + ->runs[0] + ->text, + "url" => + "https://www.youtube.com/channel/" . + $video + ->longBylineText + ->runs[0] + ->navigationEndpoint + ->browseEndpoint + ->browseId, + "avatar" => + $this->checkhttpspresence( + $video + ->channelThumbnailSupportedRenderers + ->channelThumbnailWithLinkRenderer + ->thumbnail + ->thumbnails[0] + ->url + ) + ], + "date" => $date, + "duration" => $duration, + "views" => $views, + "thumb" => [ + "url" => + $video + ->thumbnail + ->thumbnails[0] + ->url, + "ratio" => "16:9" + ], + "url" => + "https://www.youtube.com/watch?v=" . + $video + ->videoId + ] + ); + }elseif(isset($video->watchCardCompactVideoRenderer)){ + + $video = + $video + ->watchCardCompactVideoRenderer; + + array_push( + $this->out["video"], + [ + "title" => + $video + ->title + ->simpleText, + "description" => null, + "author" => [ + "name" => + $video + ->byline + ->runs[0] + ->text, + "url" => + "https://www.youtube.com/channel/" . + $video + ->byline + ->runs[0] + ->navigationEndpoint + ->browseEndpoint + ->browseId, + "avatar" => null + ], + "date" => + $this->textualdate2unix( + trim( + explode( + "•", + $video + ->subtitle + ->simpleText, + 2 + )[1] + ) + ), + "duration" => + $this->hms2int( + $video + ->lengthText + ->simpleText + ), + "views" => + $this->truncatedcount2int( + trim( + explode( + "•", + $video + ->subtitle + ->simpleText, + 2 + )[0] + ) + ), + "thumb" => [ + "url" => + $video + ->thumbnail + ->thumbnails[0] + ->url, + "ratio" => "16:9" + ], + "url" => + "https://www.youtube.com/watch?v=" . + $video + ->navigationEndpoint + ->watchEndpoint + ->videoId + ] + ); + + }elseif(isset($video->reelShelfRenderer)){ + + foreach( + $video + ->reelShelfRenderer + ->items + as $reel + ){ + + $reel = + $reel + ->reelItemRenderer; + + array_push( + $this->out["reel"], + [ + "title" => + $reel + ->headline + ->simpleText, + "description" => null, + "author" => [ + "name" => null, + "url" => null, + "avatar" => null + ], + "date" => null, + "duration" => + $this->textualtime2int( + $reel + ->accessibility + ->accessibilityData + ->label + ), + "views" => + $this->truncatedcount2int( + $reel + ->viewCountText + ->simpleText + ), + "thumb" => [ + "url" => + $reel + ->thumbnail + ->thumbnails[0] + ->url, + "ratio" => "9:16" + ], + "url" => + "https://www.youtube.com/watch?v=" . + $reel + ->videoId + ] + ); + } + } + + elseif(isset($video->channelRenderer)){ + + $video = $video->channelRenderer; + + $description = null; + + if(isset($video->descriptionSnippet)){ + + foreach( + $video + ->descriptionSnippet + ->runs + as $description_part + ){ + + $description .= $description_part->text; + } + } + + array_push( + $this->out["author"], + [ + "title" => + $video + ->title + ->simpleText, + "followers" => + isset( + $video + ->videoCountText + ->simpleText + ) ? + $this->truncatedcount2int( + $video + ->videoCountText + ->simpleText + ) : + 0, + "description" => $this->titledots($description), + "thumb" => + [ + "url" => + $this->checkhttpspresence( + $video + ->thumbnail + ->thumbnails[ + count( + $video + ->thumbnail + ->thumbnails + ) - 1 + ] + ->url + ), + "ratio" => "1:1" + ], + "url" => + "https://www.youtube.com/channel/" . + $video + ->channelId + ] + ); + } + + elseif(isset($video->shelfRenderer)){ + + if( + !is_object( + $video + ->shelfRenderer + ->content + ->verticalListRenderer + ) + ){ + return; + } + + foreach( + $video + ->shelfRenderer + ->content + ->verticalListRenderer + ->items + as $shelfvideo + ){ + + $this->parsevideoobject($shelfvideo); + } + + }elseif(isset($video->radioRenderer)){ + + $video = $video->radioRenderer; + + $description = + $video + ->videoCountText + ->runs[0] + ->text + . "."; + + $tmp = []; + foreach( + $video->videos + as $childvideo + ){ + + $tmp[] = + $childvideo + ->childVideoRenderer + ->title + ->simpleText; + } + + if(count($tmp) !== 0){ + + $description .= + " " . implode(", ", $tmp); + } + + array_push( + $this->out["playlist"], + [ + "title" => + $video + ->title + ->simpleText, + "description" => $description, + "author" => [ + "name" => + $video + ->longBylineText + ->simpleText, + "url" => null, + "avatar" => null + ], + "date" => null, + "duration" => null, + "views" => null, + "thumb" => [ + "url" => + $video + ->thumbnail + ->thumbnails[ + count( + $video + ->thumbnail + ->thumbnails + ) - 1 + ] + ->url, + "ratio" => "16:9" + ], + "url" => + "https://www.youtube.com/watch?v=" . + $video + ->videos[0] + ->childVideoRenderer + ->videoId . + "&list=" . + $video + ->playlistId . + "&start_radio=1" + ] + ); + + }elseif(isset($video->playlistRenderer)){ + + $video = $video->playlistRenderer; + + $description = $video->videoCount . " videos."; + + $tmp = []; + foreach( + $video + ->videos + as $childvideo + ){ + + $tmp[] = + $childvideo + ->childVideoRenderer + ->title + ->simpleText; + } + + if(count($tmp) !== 0){ + + $description .= + " " . implode(", ", $tmp); + } + + array_push( + $this->out["playlist"], + [ + "title" => + $video + ->title + ->simpleText, + "description" => $description, + "author" => [ + "name" => + $video + ->longBylineText + ->runs[0] + ->text, + "url" => + "https://www.youtube.com/channel/" . + $video + ->longBylineText + ->runs[0] + ->navigationEndpoint + ->browseEndpoint + ->browseId, + "picture" => null + ], + "date" => null, + "duration" => null, + "views" => null, + "thumb" => + [ + "url" => + $video + ->thumbnails[0] + ->thumbnails[ + count( + $video + ->thumbnails[0] + ->thumbnails + ) - 1 + ] + ->url, + "ratio" => "16:9" + ], + "url" => + "https://www.youtube.com/watch?v=" . + $video + ->videos[0] + ->childVideoRenderer + ->videoId . + "&list=" . + $video + ->playlistId . + "&start_radio=1" + ] + ); + + }/*else{ + if(!isset($video->searchPyvRenderer)){ + echo json_encode($video); + die();} + }*/ + } + + private function textualdate2unix($number){ + + $number = + explode( + " ", + str_replace( + [ + " ago", + "seconds", + "minutes", + "hours", + "days", + "weeks", + "months", + "years" + ], + [ + "", + "second", + "minute", + "hour", + "day", + "week", + "month", + "year" + ], + $number + ), + 2 + ); + + $time = 0; + switch($number[1]){ + + case "second": + $time = (int)$number[0]; + break; + + case "minute": + $time = (int)$number[0] * 60; + break; + + case "hour": + $time = (int)$number[0] * 3600; + break; + + case "day": + $time = (int)$number[0] * 86400; + break; + + case "week": + $time = (int)$number[0] * 604800; + break; + + case "month": + $time = (int)$number[0] * 2629746; + break; + + case "year": + $time = (int)$number[0] * 31556952; + break; + } + + return time() - $time; + } + + private function checkhttpspresence($link){ + + if(substr($link, 0, 2) == "//"){ + + return "https:" . $link; + } + + return $link; + } + + private function textualtime2int($number){ + + $number = explode(" - ", $number); + + if(count($number) >= 2){ + + $number = $number[count($number) - 2]; + }else{ + + $number = $number[0]; + } + + $number = + str_replace( + [ + " ", + "seconds", + "minutes", + "hours", + ], + [ + "", + "second", + "minute", + "hour" + ], + $number + ); + + preg_match_all( + '/([0-9]+)(second|minute|hour)/', + $number, + $number + ); + + $time = 0; + + for($i=0; $i<count($number[0]); $i++){ + + switch($number[2][$i]){ + + case "second": + $time = $time + (int)$number[1][$i]; + break; + + case "minute": + $time = $time + ((int)$number[1][$i] * 60); + break; + + case "hour": + $time = $time + ((int)$number[1][$i] * 3600); + break; + } + } + + return $time; + } + + private function views2int($views){ + + return + (int)str_replace( + ",", "", + explode(" ", $views, 2)[0] + ); + } + + private function hms2int($time){ + + $parts = explode(":", $time, 3); + $time = 0; + + if(count($parts) === 3){ + + // hours + $time = $time + ((int)$parts[0] * 3600); + array_shift($parts); + } + + if(count($parts) === 2){ + + // minutes + $time = $time + ((int)$parts[0] * 60); + array_shift($parts); + } + + // seconds + $time = $time + (int)$parts[0]; + + return $time; + } + + private function truncatedcount2int($number){ + + // decimal should always be 1 number long + $number = explode(" ", $number, 2); + $number = $number[0]; + + $unit = strtolower($number[strlen($number) - 1]); + + $tmp = explode(".", $number, 2); + $number = (int)$number; + + if(count($tmp) === 2){ + + $decimal = (int)$tmp[1]; + }else{ + + $decimal = 0; + } + + switch($unit){ + + case "k": + $exponant = 1000; + break; + + case "m": + $exponant = 1000000; + break; + + case "b"; + $exponant = 1000000000; + break; + + default: + $exponant = 1; + break; + } + + return ($number * $exponant) + ($decimal * ($exponant / 10)); + } + + private function titledots($title){ + + $substr = substr($title, -3); + + if( + $substr == "..." || + $substr == "…" + ){ + + return trim(substr($title, 0, -3), " \n\r\t\v\x00\0\x0B\xc2\xa0"); + } + + return trim($title, " \n\r\t\v\x00\0\x0B\xc2\xa0"); + } +} diff --git a/settings.php b/settings.php new file mode 100644 index 0000000..29f051d --- /dev/null +++ b/settings.php @@ -0,0 +1,316 @@ +<?php + +/* + Define settings +*/ +$settings = [ + [ + "name" => "General", + "settings" => [ + [ + "description" => "Allow NSFW content", + "parameter" => "nsfw", + "options" => [ + [ + "value" => "yes", + "text" => "Yes" + ], + [ + "value" => "maybe", + "text" => "Maybe" + ], + [ + "value" => "no", + "text" => "No" + ] + ] + ], + [ + "description" => "Theme", + "parameter" => "theme", + "options" => [ + [ + "value" => "dark", + "text" => "Gruvbox dark" + ], + [ + "value" => "cream", + "text" => "Gruvbox cream" + ] + ] + ], + [ + "description" => "Prevent clicking background elements when image viewer is open", + "parameter" => "bg_noclick", + "options" => [ + [ + "value" => "no", + "text" => "No" + ], + [ + "value" => "yes", + "text" => "Yes" + ] + ] + ] + ] + ], + [ + "name" => "Scrapers to use", + "settings" => [ + [ + "description" => "Web", + "parameter" => "scraper_web", + "options" => [ + [ + "value" => "ddg", + "text" => "DuckDuckGo" + ], + [ + "value" => "brave", + "text" => "Brave" + ], + [ + "value" => "google", + "text" => "Google" + ], + [ + "value" => "mojeek", + "text" => "Mojeek" + ], + [ + "value" => "marginalia", + "text" => "Marginalia" + ], + [ + "value" => "wiby", + "text" => "wiby" + ] + ] + ], + [ + "description" => "Images", + "parameter" => "scraper_images", + "options" => [ + [ + "value" => "ddg", + "text" => "DuckDuckGo" + ], + [ + "value" => "yandex", + "text" => "Yandex" + ], + [ + "value" => "google", + "text" => "Google" + ] + ] + ], + [ + "description" => "Videos", + "parameter" => "scraper_videos", + "options" => [ + [ + "value" => "yt", + "text" => "YouTube" + ], + [ + "value" => "ddg", + "text" => "DuckDuckGo" + ], + [ + "value" => "google", + "text" => "Google" + ] + ] + ], + [ + "description" => "News", + "parameter" => "scraper_news", + "options" => [ + [ + "value" => "ddg", + "text" => "DuckDuckGo" + ], + [ + "value" => "brave", + "text" => "Brave" + ], + [ + "value" => "google", + "text" => "Google" + ], + [ + "value" => "mojeek", + "text" => "Mojeek" + ] + ] + ] + ] + ] +]; + +/* + Set cookies +*/ + +if($_POST){ + + $loop = &$_POST; +}else{ + + // refresh cookie dates + $loop = &$_COOKIE; +} + +foreach($loop as $key => $value){ + + foreach($settings as $title){ + + foreach($title["settings"] as $list){ + + if( + $list["parameter"] == $key && + $list["options"][0]["value"] == $value + ){ + + unset($_COOKIE[$key]); + + setcookie( + $key, + "", + [ + "expires" => -1, // removes cookie + "samesite" => "Strict" + ] + ); + + continue 3; + } + } + } + + if(!is_string($value)){ + + continue; + } + + $key = trim($key); + $value = trim($value); + + $_COOKIE[$key] = $value; + + setcookie( + $key, + $value, + [ + "expires" => strtotime("+400 days"), // maximal cookie ttl in chrome + "samesite" => "Strict" + ] + ); +} + +include "lib/frontend.php"; +$frontend = new frontend(); + +echo + '<!DOCTYPE html>' . + '<html lang="en">' . + '<head>' . + '<meta http-equiv="Content-Type" content="text/html;charset=utf-8">' . + '<title>Settings</title>' . + '<link rel="stylesheet" href="/static/style.css">' . + '<meta name="viewport" content="width=device-width,initial-scale=1">' . + '<meta name="robots" content="index,follow">' . + '<link rel="icon" type="image/x-icon" href="/favicon.ico">' . + '<meta name="description" content="4get.ca: Settings">' . + '<link rel="search" type="application/opensearchdescription+xml" title="4get" href="/opensearch.xml">' . + '</head>' . + '<body' . $frontend->getthemeclass() . '>'; + +$left = + '<h1>Settings</h1>' . + '<form method="post" autocomplete="off">' . + 'By clicking <div class="code-inline">Update settings!</div>, a plaintext <div class="code-inline">key=value</div> cookie will be stored on your browser. When selecting a default setting, the parameter is removed from your cookies.'; + +$c = count($_COOKIE); +if($c !== 0){ + + $left .= + '<br><br>Your current cookie looks like this:' . + '<div class="code">'; + + $code = ""; + + $ca = 0; + foreach($_COOKIE as $key => $value){ + + $code .= $key . "=" . $value; + + $ca++; + if($ca !== $c){ + + $code .= "; "; + } + } + + $left .= $frontend->highlightcode($code); + + $left .= '</div>'; +}else{ + + $left .= + '<br><br>You currently don\'t have any cookies set.'; +} + +$left .= + '<div class="settings">'; + +foreach($settings as $title){ + + $left .= '<h2>' . $title["name"] . '</h2>'; + + foreach($title["settings"] as $setting){ + + $left .= + '<div class="setting">' . + '<div class="title">' . $setting["description"] . '</div>' . + '<select name="' . $setting["parameter"] . '">'; + + foreach($setting["options"] as $option){ + + $left .= + '<option value="' . $option["value"] . '"'; + + if( + isset($_COOKIE[$setting["parameter"]]) && + $_COOKIE[$setting["parameter"]] == $option["value"] + ){ + $left .= ' selected'; + } + + $left .= '>' . $option["text"] . '</option>'; + } + + $left .= '</select></div>'; + } +} + +$left .= + '</div>' . + '<div class="settings-submit">' . + '<input type="submit" value="Update settings!">' . + '<a href="../">< Return to main page</a>' . + '</div>' . + '</form>'; + +echo + $frontend->load( + "search.html", + [ + "class" => "", + "right-left" => "", + "right-right" => "", + "left" => $left + ] + ); diff --git a/sitemap.xml b/sitemap.xml new file mode 100644 index 0000000..ade4c4d --- /dev/null +++ b/sitemap.xml @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="UTF-8"?> +<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> + <url> + <loc>https://4get.ca</loc> + <lastmod>2023-07-31T07:56:12+03:00</lastmod> + </url> + <url> + <loc>https://4get.ca/about</loc> + <lastmod>2023-07-31T07:56:12+03:00</lastmod> + </url> + <url> + <loc>https://4get.ca/settings</loc> + <lastmod>2023-07-31T07:56:12+03:00</lastmod> + </url> + <url> + <loc>https://4get.ca/api.txt</loc> + <lastmod>2023-07-31T07:56:12+03:00</lastmod> + </url> +</urlset>
\ No newline at end of file diff --git a/static/client.js b/static/client.js new file mode 100644 index 0000000..82ea4df --- /dev/null +++ b/static/client.js @@ -0,0 +1,682 @@ + +/* + Global functions +*/ +function htmlspecialchars(str){ + + var map = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''' + } + + return str.replace(/[&<>"']/g, function(m){return map[m];}); +} + +function htmlspecialchars_decode(str){ + + var map = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + ''': "'" + } + + return str.replace(/&|<|>|"|'/g, function(m){return map[m];}); +} + +function is_click_within(elem, classname, is_id = false){ + + while(true){ + + if(elem === null){ + + return false; + } + + if( + ( + is_id === false && + elem.className == classname + ) || + ( + is_id === true && + elem.id == classname + ) + ){ + + return elem; + } + + elem = elem.parentElement; + } +} + + + +/* + Prevent GET parameter pollution +*/ +var form = document.getElementsByTagName("form"); + +if( + form.length !== 0 && + window.location.pathname != "/" && + window.location.pathname != "/settings.php" && + window.location.pathname != "/settings" +){ + form = form[0]; + + var scraper_dropdown = document.getElementsByName("scraper")[0]; + + scraper_dropdown.addEventListener("change", function(choice){ + + submit(form); + }); + + form.addEventListener("submit", function(e){ + + e.preventDefault(); + submit(e.srcElement); + }); +} + +function submit(e){ + + var GET = ""; + var first = true; + + if((s = document.getElementsByName("s")).length !== 0){ + + GET += "?s=" + encodeURIComponent(s[0].value).replaceAll("%20", "+"); + first = false; + } + + Array.from( + e.getElementsByTagName("select") + ).concat( + Array.from( + e.getElementsByTagName("input") + ) + ).forEach(function(el){ + + var firstelem = el.getElementsByTagName("option"); + + if( + ( + ( + firstelem.length === 0 || + firstelem[0].value != el.value + ) && + el.name != "" && + el.value != "" && + el.name != "s" + ) || + el.name == "scraper" || + el.name == "nsfw" + ){ + + if(first){ + + GET += "?"; + first = false; + }else{ + + GET += "&"; + } + + GET += encodeURIComponent(el.name).replaceAll("%20", "+") + "=" + encodeURIComponent(el.value).replaceAll("%20", "+"); + } + }); + + window.location.href = GET; +} + + + +/* + Hide show more button when it's not needed on answers +*/ +var answer_div = document.getElementsByClassName("answer"); + +if(answer_div.length !== 0){ + answer_div = Array.from(answer_div); + var spoiler_button_div = Array.from(document.getElementsByClassName("spoiler-button")); + + // execute on pageload + hide_show_more(); + + window.addEventListener("resize", hide_show_more); + + function hide_show_more(){ + + var height = window.innerWidth >= 1000 ? 600 : 200; + + for(i=0; i<answer_div.length; i++){ + + if(answer_div[i].scrollHeight < height){ + + spoiler_button_div[i].style.display = "none"; + + document.getElementById(spoiler_button_div[i].htmlFor).checked = true; + }else{ + + spoiler_button_div[i].style.display = "block"; + } + } + } +} + +switch(document.location.pathname){ + + case "/web": + case "/web.php": + var image_class = "image"; + break; + + case "/images": + case "/images.php": + var image_class = "thumb"; + break; + + default: + var image_class = null; +} + +if(image_class !== null){ + + /* + Add popup to document + */ + var popup_bg = document.createElement("div"); + popup_bg.id = "popup-bg"; + document.body.appendChild(popup_bg); + + // enable/disable pointer events + if(!document.cookie.includes("bg_noclick=yes")){ + + popup_bg.style.pointerEvents = "none"; + } + + var popup_status = document.createElement("div"); + popup_status.id = "popup-status"; + document.body.appendChild(popup_status); + + var popup_body = document.createElement("div"); + popup_body.id = "popup"; + document.body.appendChild(popup_body); + + // import popup + var popup_body = document.getElementById("popup"); + var popup_status = document.getElementById("popup-status"); + var popup_image = null; // is set later on popup click + + // image metadata + var collection = []; // will contain width, height, image URL + var collection_index = 0; + + // event handling helper variables + var is_popup_shown = false; + var mouse_down = false; + var mouse_move = false; + var move_x = 0; + var move_y = 0; + var target_is_popup = false; + var mirror_x = false; + var mirror_y = false; + var rotation = 0; + + /* + Image dragging (mousedown) + */ + document.addEventListener("mousedown", function(div){ + + if(div.buttons !== 1){ + + return; + } + + mouse_down = true; + mouse_move = false; + + if(is_click_within(div.target, "popup", true) === false){ + + target_is_popup = false; + }else{ + + target_is_popup = true; + + var pos = popup_body.getBoundingClientRect(); + move_x = div.x - pos.x; + move_y = div.y - pos.y; + } + }); + + /* + Image dragging (mousemove) + */ + document.addEventListener("mousemove", function(pos){ + + if( + target_is_popup && + mouse_down + ){ + + mouse_move = true; + movepopup(popup_body, pos.clientX - move_x, pos.clientY - move_y); + } + }); + + /* + Image dragging (mouseup) + */ + document.addEventListener("mouseup", function(){ + + mouse_down = false; + }); + + /* + Image popup open + */ + document.addEventListener("click", function(click){ + + // should our click trigger image open? + if( + elem = is_click_within(click.target, image_class) || + click.target.classList.contains("openimg") + ){ + + event.preventDefault(); + is_popup_shown = true; + + // reset position params + mirror_x = false; + mirror_y = false; + rotation = 0; + scale = 60; + collection_index = 0; + + // get popup data + if(elem === true){ + // we clicked a simple image preview + elem = click.target; + var image_url = elem.getAttribute("src"); + + if(image_url.startsWith("/proxy")){ + + var match = image_url.match(/i=([^&]+)/); + + if(match !== null){ + + image_url = decodeURIComponent(match[1]); + } + }else{ + + image_url = htmlspecialchars_decode(image_url); + } + + collection = [ + { + "url": image_url, + "width": Math.round(click.target.naturalWidth), + "height": Math.round(click.target.naturalHeight) + } + ]; + + var title = "No description provided"; + + if(click.target.title != ""){ + + title = click.target.title; + }else{ + + if(click.target.alt != ""){ + + title = click.target.alt; + } + } + }else{ + + if(image_class == "thumb"){ + // we're inside image.php + + elem = + elem + .parentElement + .parentElement; + + var image_url = elem.getElementsByTagName("a")[1].href; + }else{ + + // we're inside web.php + var image_url = elem.href; + } + + collection = + JSON.parse( + elem.getAttribute("data-json") + ); + + var title = elem.title; + } + + // prepare HTML + var html = + '<div id="popup-num">(' + collection.length + ')</div>' + + '<div id="popup-dropdown">' + + '<select name="viewer-res" onchange="changeimage(event)">'; + + for(i=0; i<collection.length; i++){ + + if(collection[i].url.startsWith("data:")){ + + var domain = "<Base64 Data>"; + }else{ + + var domain = new URL(collection[i].url).hostname; + } + + html += '<option value="' + i + '">' + '(' + collection[i].width + 'x' + collection[i].height + ') ' + domain + '</option>'; + } + + popup_status.innerHTML = + html + '</select></div>' + + '<a href="' + htmlspecialchars(image_url) + '" rel="noreferrer nofollow "id="popup-title">' + htmlspecialchars(title) + '</a>'; + + popup_body.innerHTML = + '<img src="' + getproxylink(collection[0].url) + '" draggable="false" id="popup-image">'; + + // make changes to DOM + popup_body.style.display = "block"; + popup_bg.style.display = "block"; + popup_status.style.display = "table"; + + // store for rotation functions & changeimage() + popup_image = document.getElementById("popup-image"); + + scalepopup(collection[collection_index], scale); + centerpopup(); + }else{ + + // click inside the image viewer + // resize image + if(is_click_within(click.target, "popup", true)){ + + if(mouse_move === false){ + scale = 80; + scalepopup(collection[collection_index], scale); + centerpopup(); + } + }else{ + + if(is_click_within(click.target, "popup-status", true) === false){ + + // click outside the popup while its open + // close it + if(is_popup_shown){ + + hidepopup(); + } + } + } + } + }); + + /* + Scale image viewer + */ + popup_body.addEventListener("wheel", function(scroll){ + + event.preventDefault(); + + if( + scroll.altKey || + scroll.ctrlKey || + scroll.shiftKey + ){ + + var increment = 7; + }else{ + + var increment = 14; + } + + if(scroll.wheelDelta > 0){ + + // scrolling up + scale = scale + increment; + }else{ + + // scrolling down + if(scale - increment > 7){ + scale = scale - increment; + } + } + + // calculate relative size before scroll + var pos = popup_body.getBoundingClientRect(); + var x = (scroll.x - pos.x) / pos.width; + var y = (scroll.y - pos.y) / pos.height; + + scalepopup(collection[collection_index], scale); + + // move popup to % we found + pos = popup_body.getBoundingClientRect(); + + movepopup( + popup_body, + scroll.clientX - (x * pos.width), + scroll.clientY - (y * pos.height) + ); + }); + + /* + Keyboard controls + */ + + document.addEventListener("keydown", function(key){ + + // close popup + if( + is_popup_shown && + key.keyCode === 27 + ){ + + hidepopup(); + return; + } + + if(is_popup_shown === false){ + + return; + } + + if( + key.altKey || + key.ctrlKey || + key.shiftKey + ){ + + // mirror image + switch(key.keyCode){ + + case 37: + // left + key.preventDefault(); + mirror_x = true; + break; + + case 38: + // up + key.preventDefault(); + mirror_y = false; + break; + + case 39: + // right + key.preventDefault(); + mirror_x = false; + break; + + case 40: + // down + key.preventDefault(); + mirror_y = true; + break; + } + }else{ + + // rotate image + switch(key.keyCode){ + + case 37: + // left + key.preventDefault(); + rotation = -90; + break; + + case 38: + // up + key.preventDefault(); + rotation = 0; + break; + + case 39: + // right + key.preventDefault(); + rotation = 90; + break; + + case 40: + // down + key.preventDefault(); + rotation = -180; + break; + } + } + + popup_image.style.transform = + "scale(" + + (mirror_x ? "-1" : "1") + + ", " + + (mirror_y ? "-1" : "1") + + ") " + + "rotate(" + + rotation + "deg" + + ")"; + }); +} + +function getproxylink(url){ + + if(url.startsWith("data:")){ + + return htmlspecialchars(url); + }else{ + + console.log(url); + return '/proxy?i=' + encodeURIComponent(url); + } +} + +function hidepopup(){ + + is_popup_shown = false; + popup_status.style.display = "none"; + popup_body.style.display = "none"; + popup_bg.style.display = "none"; +} + +function scalepopup(size, scale){ + + var ratio = + Math.min( + (window.innerWidth * (scale / 100)) / collection[collection_index].width, (window.innerHeight * (scale / 100)) / collection[collection_index].height + ); + + popup_body.style.width = size.width * ratio + "px"; + popup_body.style.height = size.height * ratio + "px"; +} + +function centerpopup(){ + + var size = popup_body.getBoundingClientRect(); + var size = { + "width": parseInt(size.width), + "height": parseInt(size.height) + }; + + movepopup( + popup_body, + (window.innerWidth / 2) - (size.width / 2), + (window.innerHeight / 2) - (size.height / 2) + ); +} + +function movepopup(popup_body, x, y){ + + popup_body.style.left = x + "px"; + popup_body.style.top = y + "px"; +} + +function changeimage(event){ + + // reset rotation params + mirror_x = false; + mirror_y = false; + rotation = 0; + + scale = 60; + + collection_index = parseInt(event.target.value); + + // we set innerHTML otherwise old image lingers a little + popup_body.innerHTML = + '<img src="' + getproxylink(collection[collection_index].url) + '" draggable="false" id="popup-image">'; + + // store for rotation functions & changeimage() + popup_image = document.getElementById("popup-image"); + + scalepopup(collection[collection_index], scale); + centerpopup(); +} + +/* + Shortcuts +*/ +var searchbox_wrapper = document.getElementsByClassName("searchbox"); + +if(searchbox_wrapper.length !== 0){ + searchbox_wrapper = searchbox_wrapper[0]; + var searchbox = searchbox_wrapper.getElementsByTagName("input")[1]; + + document.addEventListener("keydown", function(key){ + + switch(key.keyCode){ + + case 191: + // 191 = / + if(document.activeElement.tagName == "INPUT"){ + + // already focused, ignore + break; + } + + if( + typeof is_popup_shown != "undefined" && + is_popup_shown + ){ + + hidepopup(); + } + + window.scrollTo(0, 0); + searchbox.focus(); + key.preventDefault(); + break; + } + }); +} diff --git a/static/icon/amazon.png b/static/icon/amazon.png Binary files differnew file mode 100644 index 0000000..75b52a7 --- /dev/null +++ b/static/icon/amazon.png diff --git a/static/icon/appstore.png b/static/icon/appstore.png Binary files differnew file mode 100644 index 0000000..46a0be0 --- /dev/null +++ b/static/icon/appstore.png diff --git a/static/icon/facebook.png b/static/icon/facebook.png Binary files differnew file mode 100644 index 0000000..e40590c --- /dev/null +++ b/static/icon/facebook.png diff --git a/static/icon/gamespot.png b/static/icon/gamespot.png Binary files differnew file mode 100644 index 0000000..a5b781e --- /dev/null +++ b/static/icon/gamespot.png diff --git a/static/icon/github.png b/static/icon/github.png Binary files differnew file mode 100644 index 0000000..f0455f4 --- /dev/null +++ b/static/icon/github.png diff --git a/static/icon/googleplay.png b/static/icon/googleplay.png Binary files differnew file mode 100644 index 0000000..f59ca3d --- /dev/null +++ b/static/icon/googleplay.png diff --git a/static/icon/imdb.png b/static/icon/imdb.png Binary files differnew file mode 100644 index 0000000..8ddb803 --- /dev/null +++ b/static/icon/imdb.png diff --git a/static/icon/instagram.png b/static/icon/instagram.png Binary files differnew file mode 100644 index 0000000..f1bf8e6 --- /dev/null +++ b/static/icon/instagram.png diff --git a/static/icon/itunes.png b/static/icon/itunes.png Binary files differnew file mode 100644 index 0000000..cf99ab4 --- /dev/null +++ b/static/icon/itunes.png diff --git a/static/icon/microsoft.png b/static/icon/microsoft.png Binary files differnew file mode 100644 index 0000000..830fadf --- /dev/null +++ b/static/icon/microsoft.png diff --git a/static/icon/quora.png b/static/icon/quora.png Binary files differnew file mode 100644 index 0000000..2bcbb48 --- /dev/null +++ b/static/icon/quora.png diff --git a/static/icon/reddit.png b/static/icon/reddit.png Binary files differnew file mode 100644 index 0000000..8f24212 --- /dev/null +++ b/static/icon/reddit.png diff --git a/static/icon/rottentomatoes.png b/static/icon/rottentomatoes.png Binary files differnew file mode 100644 index 0000000..01f553a --- /dev/null +++ b/static/icon/rottentomatoes.png diff --git a/static/icon/sciencedirect.png b/static/icon/sciencedirect.png Binary files differnew file mode 100644 index 0000000..17e03c0 --- /dev/null +++ b/static/icon/sciencedirect.png diff --git a/static/icon/soundcloud.png b/static/icon/soundcloud.png Binary files differnew file mode 100644 index 0000000..9c86c48 --- /dev/null +++ b/static/icon/soundcloud.png diff --git a/static/icon/spotify.png b/static/icon/spotify.png Binary files differnew file mode 100644 index 0000000..07fd1f3 --- /dev/null +++ b/static/icon/spotify.png diff --git a/static/icon/steam.png b/static/icon/steam.png Binary files differnew file mode 100644 index 0000000..749ba3e --- /dev/null +++ b/static/icon/steam.png diff --git a/static/icon/twitter.png b/static/icon/twitter.png Binary files differnew file mode 100644 index 0000000..a6d4389 --- /dev/null +++ b/static/icon/twitter.png diff --git a/static/icon/w3html.png b/static/icon/w3html.png Binary files differnew file mode 100644 index 0000000..c010842 --- /dev/null +++ b/static/icon/w3html.png diff --git a/static/icon/website.png b/static/icon/website.png Binary files differnew file mode 100644 index 0000000..02b2670 --- /dev/null +++ b/static/icon/website.png diff --git a/static/icon/wikipedia.png b/static/icon/wikipedia.png Binary files differnew file mode 100644 index 0000000..18cf9a0 --- /dev/null +++ b/static/icon/wikipedia.png diff --git a/static/icon/youtube.png b/static/icon/youtube.png Binary files differnew file mode 100644 index 0000000..6951c84 --- /dev/null +++ b/static/icon/youtube.png diff --git a/static/style.css b/static/style.css new file mode 100644 index 0000000..c7cacfe --- /dev/null +++ b/static/style.css @@ -0,0 +1,1176 @@ + +/* + Global styles +*/ +:root{ + /* background */ + --1d2021: #1d2021; + --282828: #282828; + --3c3836: #3c3836; + --504945: #504945; + + /* font */ + --928374: #928374; + --a89984: #a89984; + --bdae93: #bdae93; + --8ec07c: #8ec07c; + --ebdbb2: #ebdbb2; + + /* code highlighter */ + --comment: #9e8e73; + --default: #d4be98; + --keyword: #d8a657; + --string: #7daea7; +} + +.theme-white{ + /* background */ + --1d2021: #bdae93; + --282828: #a89984; + --3c3836: #a89984; + --504945: #504945; + + /* font */ + --928374: #1d2021; + --a89984: #282828; + --bdae93: #3c3836; + --8ec07c: #52520e; + --ebdbb2: #1d2021; + + /* code highlighter */ + --comment: #6a4400; + --default: #d4be98; + --keyword: #4a4706; + --string: #076678; +} + +.theme-white .autocomplete .entry:hover{ + background:#928374; +} + +audio{ + max-width:100%; +} + +body,html{ + padding:0; + margin:0; +} + +body{ + background:var(--1d2021); + color:var(--a89984); + font-size:16px; + box-sizing:border-box; + font-family:sans-serif; + padding:15px 7% 45px; + word-wrap:anywhere; + line-height:22px; + max-width:2000px; +} + +h1,h2,h3,h4,h5,h6{ + padding:0; + margin:0 0 7px 0; + line-height:initial; + color:var(--bdae93); +} + +h3,h4,h5,h6{ + margin-bottom:14px; +} + +/* + Web styles +*/ +#overflow{ + overflow:hidden; +} + +/* Searchbox */ +.searchbox{ + width:40%; + height:36px; + border:1px solid var(--504945); + background:var(--282828); + border-radius:2px; + margin-bottom:10px; + position:relative; +} + +.searchbox .wrapper{ + overflow:hidden; +} + +.searchbox input[type="text"]{ + width:100%; + padding-left:10px; +} + +.searchbox input[type="text"]::placeholder{ + color:var(--928374); +} + +.searchbox input[type="submit"]{ + float:right; + cursor:pointer; + padding:0 10px; +} + +.searchbox input[type="submit"]:hover{ + text-decoration:underline; +} + +.searchbox input{ + all:unset; + line-height:36px; + box-sizing:border-box; + height:36px; + color:var(--bdae93); +} + +.searchbox:focus-within{ + border:1px solid var(--928374); +} + +.autocomplete{ + display:none; + position:absolute; + top:35px; + left:-1px; + right:-1px; + background:var(--282828); + border:1px solid var(--504945); + border-top:none; + border-radius:0 0 2px 2px; + z-index:10; +} + +.autocomplete .entry{ + overflow:hidden; + padding:4px 10px; + cursor:pointer; +} + +.autocomplete .entry:hover{ + background:var(--3c3836); +} + +.autocomplete .title{ + float:left; +} + +.autocomplete .subtext{ + float:right; + font-size:14px; + color:var(--928374); + margin-left:7px; +} + +/* Tabs */ +.tabs, .filters{ + overflow:hidden; + overflow-x:auto; + white-space:nowrap; +} + +.tabs{ + padding-bottom:10px; +} + +.tabs .tab{ + text-decoration:none; + color:var(--bdae93); + padding:4px 10px; + display:inline-block; +} + +.tabs .tab:hover{ + text-decoration:underline; +} + +.tabs .tab.selected{ + border-bottom:2px solid var(--bdae93); +} + +/* Filters */ +.filters{ + padding-bottom:15px; + margin-bottom:7px; +} + +.filters .filter{ + display:inline-block; + margin-right:7px; + vertical-align:bottom; +} + +.filters .filter .title{ + font-size:13px; + margin:0 4px; +} + +.filters .filter input, +.filters .filter select{ + all:unset; + display:block; + border:1px solid var(--504945); + border-radius:2px; + font-size:14px; + padding:0 4px; + width:127px; + height:24px; +} + + +/* + HOME +*/ +.home{ + min-height:100vh; + margin:0 auto; + display:table; + text-align:center; +} + +.home .logo{ + max-width:400px; + height:100px; + margin:0 auto 17px auto; +} + +.home img{ + line-height:100px; + font-size:60px; + text-align:center; + font-family:Times; + width:100%; + height:100%; + background:var(--282828); + display:block; + object-fit:contain; +} + +.home #center{ + display:table-cell; + vertical-align:middle; + width:500px; +} + +.home .searchbox{ + width:100%; + text-align:left; + margin-bottom:20px; +} + +.home a{ + color:inherit; +} + +.home .subtext{ + margin-top:17px; + line-height:16px; + font-size:12px; +} + + +/* + WEB +*/ + +/* Left wrapper */ +.web .left{ + width:40%; + float:left; +} + +/* infobox */ +.infobox{ + border:1px dashed var(--504945); + padding:10px; + margin-bottom:17px; +} + +.infobox .code{ + white-space:initial; +} + +.infobox ul{ + padding-left:27px; + margin-bottom:0; +} + +.infobox a{ + color:var(--bdae93); +} + +.infobox a:hover{ + text-decoration:underline; +} + +/* text-result */ +.web .text-result{ + margin-bottom:30px; +} + +.web .description{ + white-space:pre-line; +} + +.web .type{ + border:1px solid var(--928374); + background:var(--282828); + padding:0 4px; + border-radius:2px; + font-size:14px; + line-height:16px; + float:left; + margin:2px 7px 0 0; +} + +.web .url{ + position:relative; +} + +.web .url .part{ + font-size:15px; + text-decoration:none; + color:var(--928374); +} + +.web .separator::before{ + content:"/"; + padding:0 4px; + color:var(--504945); + font-size:12px; +} + +.web .part:hover{ + text-decoration:underline; +} + +.web .hover{ + display:block; + text-decoration:none; + color:var(--a89984); + overflow:hidden; + clear:left; + padding-top:7px; +} + +.web .text-result .title{ + font-size:18px; + color:var(--bdae93); + margin-bottom:7px; +} + +.web .text-result a:visited .title{ + color:var(--928374) !important; +} + +.theme-white .web .text-result a:visited .title{ + color:#7c6f64 !important; +} + +.web .text-result .hover:hover .title{ + text-decoration:underline; +} + +.web .text-result .author{ + font-style:italic; +} + +.web .text-result .greentext{ + font-size:14px; + color:var(--8ec07c); +} + +.web .right-right .text-result:last-child, +.web .right-left .text-result:last-child{ + margin-bottom:0; +} + +/* favicon */ +.favicon{ + all:unset; + float:left; + cursor:pointer; +} + +.favicon-dropdown{ + display:none; + position:absolute; + top:25px; + background:var(--282828); + border:1px solid var(--504945); + border-radius:2px; + z-index:3; + word-wrap:normal; +} + +.favicon-dropdown::before{ + content:""; + position:absolute; + top:-10px; + left:2px; + border:5px solid transparent; + border-bottom:5px solid var(--504945); +} + +.favicon-dropdown a{ + text-decoration:none; + color:var(--bdae93); + display:block; + padding:2px 7px 2px 5px; + font-size:13px; +} + +.favicon-dropdown a:hover{ + text-decoration:underline; +} + +.favicon-dropdown:hover, +.favicon:focus + .favicon-dropdown, +.favicon-dropdown:focus-within{ + display:block; +} + +.web .favicon img, +.favicon-dropdown img{ + margin:3px 7px 0 0; + height:16px; + font-size:12px; + line-height:16px; + text-align:center; + display:block; + text-align:left; +} + +.favicon-dropdown img{ + float:left; + margin:2px 7px 0 0; +} + +/* thumbnails */ +.thumb-wrap{ + position:relative; + float:right; + width:160px; + height:90px; + background:var(--282828); + text-align:center; + line-height:87px; + border:1px solid var(--504945); + overflow:hidden; + margin-left:7px; +} + +.duration{ + position:absolute; + right:0; + bottom:0; + padding:1px 2px; + line-height:14px; + background:var(--3c3836); + font-size:12px; + border-left:1px solid var(--504945); + border-top:1px solid var(--504945); + font-family:monospace; +} + +.web .text-result:hover .thumb-wrap .duration{ + display:none; +} + +.thumb-wrap .thumb{ + max-width:100%; + max-height:100%; + text-align:left; + vertical-align:middle; +} + +.thumb-wrap.portrait{ + width:50px; +} + +.thumb-wrap.square{ + width:90px; +} + +/* Next page */ +.nextpage{ + margin:0 0 30px; + text-align:center; + display:block; + padding:10px; + border:1px solid var(--504945); + border-radius:2px; + text-decoration:none; + color:var(--bdae93); +} + +.nextpage:hover{ + text-decoration:underline; +} + +/* Right wrapper */ +.web .right-wrapper{ + width:60%; + float:right; + overflow:hidden; + padding-left:15px; + box-sizing:border-box; +} + +.web .right-right, +.web .right-left{ + float:right; + width:50%; + padding:0 15px; + box-sizing:border-box; + overflow:hidden; + min-height:1px; +} + +.web .right-right{ + padding-right:0; +} + +/* + Tables +*/ +table{ + width:100%; + text-align:left; + border-collapse:collapse; +} + +table td{ + width:50%; + padding:0; + vertical-align:top; +} + +table tr td:first-child{ + padding-right:7px; +} + +table a{ + display:block; + text-decoration:none; + color:var(--a89984); + padding:0 10px 0 0; +} + +table tr a:last-child{ + padding-right:0; +} + +/* Related */ +.related{ + margin-bottom:20px; +} + +.related a{ + padding-bottom:10px; + color:var(--bdae93); +} + +.related a:hover{ + text-decoration:underline; +} + +/* + Answers +*/ +.web .answer{ + max-height:600px; + overflow:hidden; + padding-bottom:17px; + position:relative; +} + +.web .answer::after{ + content:""; + position:absolute; + bottom:0; + width:100%; + height:17px; + background:linear-gradient(transparent, var(--1d2021)); + pointer-events:none; +} + +.web .answer-title{ + text-decoration:none; + color:var(--a89984); +} + +.web .answer-title a:hover{ + text-decoration:underline; +} + +.web .spoiler:checked + .answer{ + overflow:initial; + max-height:initial; +} + +.web .spoiler{ + display:none; +} + +.web .spoiler-button{ + display:block; + border:1px solid var(--504945); + border-radius:2px; + line-height:30px; + padding:0 7px; + text-align:center; + cursor:pointer; +} + +.web .answer-wrapper{ + margin-bottom:27px; +} + +.web .spoiler-button:hover{ + text-decoration:underline; +} + +.web .spoiler-button::before{ + content:"Show more"; +} + +.web .spoiler:checked + .answer + .spoiler-button::before{ + content:"Show less"; +} + +/* Tables on left handside */ +.web .info-table{ + margin:10px 0; + font-size:15px; + color:var(--928374); + background:var(--282828); + border:1px dashed var(--504945); +} + +.web .info-table td{ + padding:4px 10px; +} + +.web .info-table tr td:first-child{ + width:1%; + white-space:nowrap; + padding-right:17px; + color:var(--a89984); +} + +.web .info-table tr:nth-child(even){ + background:var(--1d2021); +} + +.web .sublinks{ + padding:17px 10px 0; + font-size:15px; + color:var(--#928374); +} + +.web .sublinks table td{ + padding-bottom:17px; +} + +.web .sublinks table tr:last-child td:last-child{ + padding-bottom:0; +} + +.web .sublinks a:hover .title{ + text-decoration:underline; +} + +/* Wikipedia head */ +.web .wiki-head .photo{ + float:right; + margin:0 1px 10px 10px; +} +.web .wiki-head .photo img{ + display:block; + max-width:200px; + max-height:200px; + filter:drop-shadow(1px 0 0 var(--504945)) drop-shadow(-1px 0 0 var(--504945)) drop-shadow(0 -1px 0 var(--504945)) drop-shadow(0 1px 0 var(--504945)); +} + +.web .wiki-head .description{ + clear:left; + padding-top:7px; + overflow:hidden; +} + +.web .wiki-head table, .about table{ + margin-top:17px; + border:1px dashed var(--504945); +} + +.web .wiki-head td, .about table td{ + padding:4px 7px; + vertical-align:middle; +} + +.web .wiki-head tr td:first-child, .about table tr td:first-child{ + width:30%; + min-width:150px; +} + +.web .wiki-head tr:nth-child(odd), .about table tr:nth-child(odd){ + background:var(--282828); +} + +.web .wiki-head .socials{ + overflow:hidden; + margin-top:17px; +} + +.web .wiki-head .socials a{ + width:74px; + height:80px; + padding-right:4px; + float:left; + color:var(--bdae93); + text-decoration:none; + display:table; +} + +.web .wiki-head .socials a:hover .title{ + text-decoration:underline; +} + +.web .wiki-head .socials .center{ + display:table-cell; + vertical-align:middle; +} + +.web .wiki-head .socials img{ + margin:0 auto; + display:block; + text-align:center; + width:36px; + height:36px; + line-height:36px; +} + +.web .wiki-head .socials .title{ + margin-top:7px; + text-align:center; + font-size:13px; + line-height:13px; +} + +.web .fullimg{ + display:block; + max-width:100%; + max-height:150px; + margin:7px 0 17px; + box-sizing:border-box; + border:1px solid var(--504945); +} + +/* + Code tags +*/ +.code{ + white-space:pre; + font-family:monospace; + background:var(--3c3836); + color:var(--bdae93); + padding:7px; + margin:4px 0 13px 0; + overflow-x:auto; + border-radius:2px; + border:1px solid var(--504945); +} + +.code-inline{ + display:inline; + font-family:monospace; + background:var(--282828); + color:var(--bdae93); + border:1px solid var(--928374); + padding:0 4px; + border-radius:2px; +} + +/* Wiki-head titles and quotes */ +.web .wiki-head h2{ + font-size:20px; + margin:20px 0 13px 0; +} + +.web .wiki-head h2:first-child{ + margin-top:10px; +} + +.web .wiki-head a{ + color:var(--bdae93); +} + +.quote{ + font-style:italic; + margin:10px 0 13px; + padding-left:10px; + border-left:1px solid #504945; +} + +/* + Web images +*/ +.web .images{ + overflow:hidden; + margin:0 -5px; + font-size:0; +} + +.web .images .duration{ + display:none; + border:1px solid var(--504945); + right:5px; + bottom:5px; +} + +.web .images .image:hover .duration{ + display:block; +} + +.web .images .image{ + width:90px; + height:90px; + padding:5px; + position:relative; + line-height:90px; + display:inline-block; + text-align:center; + color:inherit; +} + +.web .images .image img{ + max-width:100%; + max-height:100%; + vertical-align:middle; +} + + +/* + Images tab +*/ + +#images{ + overflow:hidden; + margin-bottom:10px; +} + +#images .infobox{ + width:40%; + box-sizing:border-box; +} + +#images .image-wrapper{ + line-height:15px; + width:20%; + float:left; +} + +#images .image{ + margin:0 auto; + width:250px; + max-width:100%; + padding:7px 7px 30px 7px; + box-sizing:border-box; + font-size:14px; +} + +#images a{ + color:inherit; + text-decoration:none; + display:block; +} + +#images a:hover .title{ + text-decoration:underline; +} + +#images .thumb{ + display:block; + height:180px; + margin-bottom:10px; + position:relative; +} + +#images .duration{ + display:block; + border:1px solid #504945; +} + +#images .image:hover .duration{ + display:none; +} + +#images img{ + width:100%; + height:100%; + object-fit:contain; +} + +#images .image .title{ + white-space:nowrap; + overflow:hidden; + margin-bottom:7px; + font-weight:bold; +} + +#images .image .description{ + overflow:hidden; + height:45px; +} + +.nextpage.img{ + width:50%; + margin:0 auto 50px; +} + +#popup{ + display:none; + position:fixed; + top:0; + left:0; + cursor:grab; + user-select:none; + pointer-events:none; +} + +#popup:active{ + cursor:grabbing; +} + +#popup-image{ + border:1px solid var(--928374); + display:block; + margin:0 auto; + pointer-events:all; + width:100%; + height:100%; + object-fit:contain; + background:var(--282828); +} + +#popup-status{ + display:none; + position:fixed; + top:0; + left:0; + width:100%; + height:35px; + background:var(--1d2021); + border-bottom:1px solid var(--928374); +} + +#popup-bg{ + background:var(--1d2021); + opacity:.5; + position:fixed; + top:0; + left:0; + width:100%; + height:100%; + display:none; +} + +#popup-status select{ + display:block; + width:250px; +} + +#popup-num, +#popup-title{ + display:table-cell; + width:0; + word-wrap:normal; + padding:0 10px; + line-height:35px; + color:var(--ebdbb2); + text-decoration:none; +} + +#popup-title:hover{ + text-decoration:underline; +} + +#popup-title{ + width:initial; + overflow:hidden; + height:35px; + display:block; +} + +#popup-num{ + font-weight:bold; +} + +#popup-dropdown{ + display:table-cell; + vertical-align:middle; + width:0; +} + +/* + Settings page +*/ +.web .settings{ + margin-top:17px; + border:1px dashed var(--504945); + padding:7px 10px 0; +} + +.web .setting{ + margin-bottom:17px; +} + +.web .setting .title{ + font-size:14px; +} + +.web .settings-submit{ + margin:17px 10px; +} + +.web .settings-submit input{ + float:right; +} + +.web .settings-submit a{ + margin-right:17px; + color:var(--bdae93); +} + +/* + About page +*/ +.about a{ + color:var(--bdae93); +} + +.about h1, .about h2{ + margin-top:17px; +} + +.about table{ + margin-bottom:17px; +} + +.about table a{ + display:inline; +} + + +/* + Syntax highlight +*/ +.c-comment{ + color:var(--comment); +} +.c-default{ + color:var(--default); +} +.c-html{ + color:var(--html); +} +.c-keyword{ + color:var(--keyword); + font-weight:bold; +} +.c-string{ + color:var(--string); +} + +/* + Responsive image +*/ +@media only screen and (max-width: 1454px){ #images .image-wrapper{ width:25%; } } +@media only screen and (max-width: 1161px){ #images .image-wrapper{ width:33.3333%; } } +@media only screen and (max-width: 750px){ #images .image-wrapper{ width:50%; } } +@media only screen and (max-width: 450px){ #images .image-wrapper{ width:100%; } } + + +/* + Responsive design +*/ +@media only screen and (max-width: 1550px){ + + .web .right-right, + .web .right-left{ + float:none; + width:initial; + padding:0 0 0 15px; + } + + .web .left, + .searchbox, + #images .infobox{ + width:60%; + } + + .web .right-wrapper{ + width:40%; + } +} + +@media only screen and (max-width: 1000px){ + + .nextpage.img{ + width:initial; + } + + .web .right-right, + .web .right-left{ + border:none; + padding:0; + } + + .web .right-wrapper{ + float:none; + padding:0; + width:initial; + } + + .web .left, + .searchbox{ + width:100%; + } + + table td{ + display:block; + width:100%; + } + + table a{ + padding:0; + } + + .web.has-answer .left::before{ + display:block; + content:"Results"; + font-size:24px; + font-weight:bold; + margin-bottom:17px; + color:var(--bdae93); + } + + .web .answer{ + max-height:200px; + } + + .web .wiki-head tr td:first-child, + .web .info-table tr td:first-child{ + text-decoration:underline; + } + + #images .infobox{ + width:100%; + } +} diff --git a/template/header.html b/template/header.html new file mode 100644 index 0000000..bd6fc8a --- /dev/null +++ b/template/header.html @@ -0,0 +1,28 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta http-equiv="Content-Type" content="text/html;charset=utf-8"> + <title>{%title%}</title> + <link rel="stylesheet" href="/static/style.css"> + <meta name="viewport" content="width=device-width,initial-scale=1"> + <meta name="robots" content="{%index%}index,{%index%}follow"> + <link rel="icon" type="image/x-icon" href="/favicon.ico"> + <meta name="description" content="4get.ca: {%description%}"> + <link rel="search" type="application/opensearchdescription+xml" title="4get" href="/opensearch.xml"> + </head> + <body{%body_class%}> + <form method="GET" autocomplete="off"> + <div class="searchbox"> + <input type="submit" value="Search" tabindex="-1"> + <div class="wrapper"> + <input type="text" value="{%search%}" maxlength="500" name="s" placeholder="Proxy search..." required> + </div> + <div class="autocomplete"></div> + </div> + <div class="tabs"> + {%tabs%} + </div> + <div class="filters"> + {%filters%} + </div> + </form> diff --git a/template/home.html b/template/home.html new file mode 100644 index 0000000..7f00dae --- /dev/null +++ b/template/home.html @@ -0,0 +1,36 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta http-equiv="Content-Type" content="text/html;charset=utf-8"> + <title>4get</title> + <meta name="viewport" content="width=device-width,initial-scale=1"> + <link rel="stylesheet" href="/static/style.css"> + <meta name="robots" content="index,follow"> + <link rel="icon" type="image/x-icon" href="/favicon.ico"> + <meta name="description" content="4get.ca: They live in our walls!"> + <link rel="search" type="application/opensearchdescription+xml" title="4get" href="/opensearch.xml"> + </head> + <body class="home {%body_class%}"> + <div id="center"> + <form method="GET" autocomplete="off" action="web"> + <div class="logo"> + <img src="{%banner%}" alt="4get"> + </div> + <div class="searchbox"> + <input type="submit" value="Search" tabindex="-1"> + <div class="wrapper"> + <input type="text" maxlength="500" name="s" placeholder="Proxy search..." required autofocus> + </div> + <div class="autocomplete"></div> + </div> + </form> + <a href="settings">Settings</a> • <a href="api.txt">API</a> • <a href="about">About</a> • <a href="https://git.lolcat.ca/lolcat/4get">Source</a> + <div class="subtext"> + Clearnet: <a href="https://4get.ca">4get.ca</a><br> + Tor: <a href="http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion">4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion</a><br> + Report a problem: <a href="https://lolcat.ca/contact">lolcat.ca/contact</a> + </div> + </div> + <script src="/static/client.js"></script> + </body> +</html> diff --git a/template/images.html b/template/images.html new file mode 100644 index 0000000..a09c121 --- /dev/null +++ b/template/images.html @@ -0,0 +1,7 @@ + <div id="images"> + {%images%} + </div> + {%nextpage%} + <script src="/static/client.js"></script> + </body> +</html> diff --git a/template/search.html b/template/search.html new file mode 100644 index 0000000..bbfbb54 --- /dev/null +++ b/template/search.html @@ -0,0 +1,16 @@ + <div id="overflow" class="web{%class%}"> + <div class="right-wrapper"> + <div class="right-right"> + {%right-right%} + </div> + <div class="right-left"> + {%right-left%} + </div> + </div> + <div class="left"> + {%left%} + </div> + </div> + <script src="/static/client.js"></script> + </body> +</html> diff --git a/videos.php b/videos.php new file mode 100644 index 0000000..2842703 --- /dev/null +++ b/videos.php @@ -0,0 +1,241 @@ +<?php + +/* + Initialize random shit +*/ +include "lib/frontend.php"; +$frontend = new frontend(); + +[$scraper, $filters] = $frontend->getscraperfilters("videos"); + +$get = $frontend->parsegetfilters($_GET, $filters); + +$frontend->loadheader( + $get, + $filters, + "videos" +); + +$payload = [ + "class" => "", + "right-left" => "", + "right-right" => "", + "left" => "" +]; + +try{ + $results = $scraper->video($get); + +}catch(Exception $error){ + + echo + $frontend->drawerror( + "Shit", + 'This scraper returned an error:' . + '<div class="code">' . htmlspecialchars($error->getMessage()) . '</div>' . + 'Things you can try:' . + '<ul>' . + '<li>Use a different scraper</li>' . + '<li>Remove keywords that could cause errors</li>' . + '<li>Use another 4get instance</li>' . + '</ul><br>' . + 'If the error persists, please <a href="/about">contact the administrator</a>.' + ); + die(); +} + +$categories = [ + "video" => "", + "author" => "", + "livestream" => "", + "playlist" => "", + "reel" => "" +]; + +/* + Set the main container +*/ +$main = null; + +if(count($results["video"]) !== 0){ + + $main = "video"; + +}elseif(count($results["playlist"]) !== 0){ + + $main = "playlist"; + +}elseif(count($results["livestream"]) !== 0){ + + $main = "livestream"; + +}elseif(count($results["author"]) !== 0){ + + $main = "author"; + +}elseif(count($results["reel"]) !== 0){ + + $main = "reel"; +}else{ + + // No results found! + echo + $frontend->drawerror( + "Nobody here but us chickens!", + 'Have you tried:' . + '<ul>' . + '<li>Using a different scraper</li>' . + '<li>Using fewer keywords</li>' . + '<li>Defining broader filters (Is NSFW turned off?)</li>' . + '</ul>' . + '</div>' + ); + die(); +} + +/* + Generate list of videos +*/ +foreach($categories as $name => $data){ + + foreach($results[$name] as $item){ + + $greentext = []; + + if( + isset($item["date"]) && + $item["date"] !== null + ){ + + $greentext[] = date("jS M y @ g:ia", $item["date"]); + } + + if( + isset($item["views"]) && + $item["views"] !== null + ){ + + $views = number_format($item["views"]); + + if($name != "livestream"){ + + $views .= " views"; + }else{ + + $views .= " watching"; + } + + $greentext[] = $views; + } + + if( + isset($item["followers"]) && + $item["followers"] !== null + ){ + + $greentext[] = number_format($item["followers"]) . " followers"; + } + + if( + isset($item["author"]["name"]) && + $item["author"]["name"] !== null + ){ + + $greentext[] = $item["author"]["name"]; + } + + $greentext = implode(" • ", $greentext); + + if( + isset($item["duration"]) && + $item["duration"] !== null + ){ + + $duration = $frontend->s_to_timestamp($item["duration"]); + }else{ + + $duration = null; + } + + $tabindex = $name == $main ? true : false; + + $categories[$name] .= $frontend->drawtextresult($item, $greentext, $duration, $get["s"], $tabindex); + } +} + +$payload["left"] = $categories[$main]; + +// dont re-draw the category +unset($categories[$main]); + +/* + Populate right handside +*/ + +$i = 1; +foreach($categories as $name => $value){ + + if($value == ""){ + + continue; + } + + if($i % 2 === 1){ + + $write = "right-left"; + }else{ + + $write = "right-right"; + } + + $payload[$write] .= + '<div class="answer-wrapper">' . + '<input id="answer' . $i . '" class="spoiler" type="checkbox">' . + '<div class="answer">' . + '<div class="answer-title">' . + '<a class="answer-title" href="?s=' . urlencode($get["s"]); + + switch($name){ + + case "playlist": + $payload[$write] .= + '&type=playlist"><h2>Playlists</h2></a>'; + break; + + case "livestream": + $payload[$write] .= + '&feature=live"><h2>Livestreams</h2></a>'; + break; + + case "author": + $payload[$write] .= + '&type=channel"><h2>Authors</h2></a>'; + break; + + case "reel": + $payload[$write] .= + '&duration=short"><h2>Reels</h2></a>'; + break; + } + + $payload[$write] .= + '</div>' . + $categories[$name] . + '</div>' . + '<label class="spoiler-button" for="answer' . $i . '"></label></div>'; + + $i++; +} + +if($i !== 1){ + + $payload["class"] = " has-answer"; +} + +if($results["npt"] !== null){ + + $payload["left"] .= + '<a href="' . $frontend->htmlnextpage($get, $results["npt"], "videos") . '" class="nextpage">Next page ></a>'; +} + +echo $frontend->load("search.html", $payload); @@ -0,0 +1,496 @@ +<?php + +/* + Initialize random shit +*/ +include "lib/frontend.php"; +$frontend = new frontend(); + +[$scraper, $filters] = $frontend->getscraperfilters("web"); + +$get = $frontend->parsegetfilters($_GET, $filters); + +$frontend->loadheader( + $get, + $filters, + "web" +); + +$payload = [ + "class" => "", + "right-left" => "", + "right-right" => "", + "left" => "" +]; + +try{ + $results = $scraper->web($get); + +}catch(Exception $error){ + + echo + $frontend->drawerror( + "Shit", + 'This scraper returned an error:' . + '<div class="code">' . htmlspecialchars($error->getMessage()) . '</div>' . + 'Things you can try:' . + '<ul>' . + '<li>Use a different scraper</li>' . + '<li>Remove keywords that could cause errors</li>' . + '<li>Use another 4get instance</li>' . + '</ul><br>' . + 'If the error persists, please <a href="/about">contact the administrator</a>.' + ); + die(); +} + +$answerlen = 0; + +/* + Spelling checker +*/ +if($results["spelling"]["type"] != "no_correction"){ + + switch($results["spelling"]["type"]){ + + case "including": + $type = "Including results for"; + break; + + case "not_many": + $type = "Not many results contains"; + break; + } + + $payload["left"] .= + '<div class="infobox">' . + $type . ' <b>' . htmlspecialchars($results["spelling"]["using"]) . '</b>.<br>' . + 'Did you mean <a href="?s=' . urlencode($results["spelling"]["correction"]) . '">' . $results["spelling"]["correction"] . '</a>?' . + '</div>'; +} + +/* + Populate links +*/ +if(count($results["web"]) === 0){ + + $payload["left"] .= + '<div class="infobox">' . + "<h1>Nobody here but us chickens!</h1>" . + 'Have you tried:' . + '<ul>' . + '<li>Using a different scraper</li>' . + '<li>Using fewer keywords</li>' . + '<li>Defining broader filters (Is NSFW turned off?)</li>' . + '</ul>' . + '</div>'; +} + +foreach($results["web"] as $site){ + + $n = null; + + if($site["date"] !== null){ + + $date = date("jS M y @ g:ia", $site["date"]); + }else{ + + $date = null; + } + + $payload["left"] .= $frontend->drawtextresult($site, $date, $n, $get["s"]); +} + +$right = []; + +/* + Generate images +*/ +if(count($results["image"]) !== 0){ + + $answerlen++; + $right["image"] = + '<div class="answer-wrapper">' . + '<input id="answer' . $answerlen . '" class="spoiler" type="checkbox">' . + '<div class="answer">' . + '<div class="answer-title">' . + '<a class="answer-title" href="/images?s=' . urlencode($get["s"]) . '"><h2>Images</h2></a>' . + '</div>' . + '<div class="images">'; + + foreach($results["image"] as $image){ + + $c = count($image["source"]) - 1; + + if( + preg_match( + '/^data:/', + $image["source"][$c]["url"] + ) + ){ + + $src = htmlspecialchars($image["source"][$c]["url"]); + }else{ + + $src = "/proxy?i=" . urlencode($image["source"][$c]["url"]) . "&s=square"; + } + + $right["image"] .= + '<a class="image" href="' . htmlspecialchars($image["url"]) . '" rel="noreferrer nofollow" title="' . htmlspecialchars($image["title"]) . '" data-json="' . htmlspecialchars(json_encode($image["source"])) . '" tabindex="-1">' . + '<img src="' . $src . '" alt="thumb">' . + '<div class="duration">' . $image["source"][0]["width"] . 'x' . $image["source"][0]["height"] . '</div>' . + '</a>'; + } + + $right["image"] .= + '</div></div>' . + '<label class="spoiler-button" for="answer' . $answerlen . '"></label></div>'; +} + +/* + Generate videos +*/ +if(count($results["video"]) !== 0){ + + $answerlen++; + $right["video"] = + '<div class="answer-wrapper">' . + '<input id="answer' . $answerlen . '" class="spoiler" type="checkbox">' . + '<div class="answer">' . + '<div class="answer-title">' . + '<a class="answer-title" href="/videos?s=' . urlencode($get["s"]) . '"><h2>Videos</h2></a>' . + '</div>'; + + foreach($results["video"] as $video){ + + if($video["views"] !== null){ + + $greentext = number_format($video["views"]) . " views"; + }else{ + + $greentext = null; + } + + if($video["date"] !== null){ + + if($greentext !== null){ + + $greentext .= " • "; + } + + $greentext .= date("jS M y @ g:ia", $video["date"]); + } + + if($video["duration"] !== null){ + + if($video["duration"] == "_LIVE"){ + + $duration = 'LIVE'; + }else{ + + $duration = $frontend->s_to_timestamp($video["duration"]); + } + }else{ + + $duration = null; + } + + $right["video"] .= $frontend->drawtextresult($video, $greentext, $duration, $get["s"], false); + } + + $right["video"] .= + '</div>' . + '<label class="spoiler-button" for="answer' . $answerlen . '"></label></div>'; +} + +/* + Generate news +*/ +if(count($results["news"]) !== 0){ + + $answerlen++; + $right["news"] = + '<div class="answer-wrapper">' . + '<input id="answer' . $answerlen . '" class="spoiler" type="checkbox">' . + '<div class="answer">' . + '<div class="answer-title">' . + '<a class="answer-title" href="/news?s=' . urlencode($get["s"]) . '"><h2>News</h2></a>' . + '</div>'; + + foreach($results["news"] as $news){ + + if($news["date"] !== null){ + + $greentext = date("jS M y @ g:ia", $news["date"]); + }else{ + + $greentext = null; + } + + $right["news"] .= $frontend->drawtextresult($news, $greentext, null, $get["s"], false); + } + + $right["news"] .= + '</div>' . + '<label class="spoiler-button" for="answer' . $answerlen . '"></label></div>'; +} + +/* + Generate answers +*/ +if(count($results["answer"]) !== 0){ + + $right["answer"] = ""; + + foreach($results["answer"] as $answer){ + + $answerlen++; + $right["answer"] .= + '<div class="answer-wrapper">' . + '<input id="answer' . $answerlen . '" class="spoiler" type="checkbox">' . + '<div class="answer"><div class="wiki-head">'; + + if(!empty($answer["title"])){ + + $right["answer"] .= + '<div class="answer-title">'; + + if(!empty($answer["url"])){ + + $right["answer"] .= '<a class="answer-title" href="' . htmlspecialchars($answer["url"]) . '" rel="noreferrer nofollow">'; + } + + $right["answer"] .= '<h1>' . htmlspecialchars($answer["title"]) . '</h1>'; + + if(!empty($answer["url"])){ + + $right["answer"] .= '</a>'; + } + + + $right["answer"] .= '</div>'; + } + + if(!empty($answer["url"])){ + + $right["answer"] .= + $frontend->drawlink($answer["url"]); + } + + $right["answer"] .= '<div class="description">'; + + if(!empty($answer["thumb"])){ + + $right["answer"] .= + '<a href="' . htmlspecialchars($answer["thumb"]) . '" rel="noreferrer nofollow" class="photo">' . + '<img src="/proxy?i=' . urlencode($answer["thumb"]) . '&s=cover" alt="thumb" class="openimg">' . + '</a>'; + } + + foreach($answer["description"] as $description){ + + switch($description["type"]){ + + case "text": + $right["answer"] .= $frontend->highlighttext($get["s"], $description["value"]); + break; + + case "title": + $right["answer"] .= + '<h2>' . + htmlspecialchars($description["value"]) . + '</h2>'; + break; + + case "italic": + $right["answer"] .= + '<i>' . + $frontend->highlighttext($get["s"], $description["value"]) . + '</i>'; + break; + + case "quote": + $right["answer"] .= + '<div class="quote">' . + $frontend->highlighttext($get["s"], $description["value"]) . + '</div>'; + break; + + case "code": + $right["answer"] .= + '<div class="code" tabindex="-1">' . + $frontend->highlightcode($description["value"], true) . + '</div>'; + break; + + case "inline_code": + $right["answer"] .= + '<div class="code-inline">' . + htmlspecialchars($description["value"]) . + '</div>'; + break; + + case "link": + $right["answer"] .= + '<a href="' . htmlspecialchars($description["url"]) . '" rel="noreferrer nofollow" class="underline" tabindex="-1">' . htmlspecialchars($description["value"]) . '</a>'; + break; + + case "image": + $right["answer"] .= + '<a href="' . htmlspecialchars($description["url"]) . '" rel="noreferrer nofollow" tabindex="-1"><img src="/proxy?i=' . urlencode($description["url"]) . '&s=thumb" alt="image" class="fullimg openimg"></a>'; + break; + + case "audio": + $right["answer"] .= + '<audio src="/audio?s=' . urlencode($description["url"]) . '" controls><a href="/audio.php?s=' . urlencode($description["url"]) . '">Listen to the pronunciation audio</a></audio>'; + break; + } + } + + $right["answer"] .= '</div>'; + + if(count($answer["table"]) !== 0){ + + $right["answer"] .= '<table>'; + + foreach($answer["table"] as $info => $value){ + + $right["answer"] .= + '<tr>' . + '<td>' . $info . '</td>' . + '<td>' . $value . '</td>' . + '</tr>'; + } + + $right["answer"] .= '</table>'; + } + + if(count($answer["sublink"]) !== 0){ + + $right["answer"] .= '<div class="socials">'; + $icons = glob("static/icon/*"); + + foreach($answer["sublink"] as $website => $url){ + + $flag = false; + $icon = str_replace(" ", "", strtolower($website)); + + foreach($icons as $path){ + + if(pathinfo($path, PATHINFO_FILENAME) == $icon){ + + $flag = true; + break; + } + } + + if($flag === false){ + + $icon = "website"; + } + + $right["answer"] .= + '<a href="' . htmlspecialchars($url) . '" rel="noreferrer nofollow" tabindex="-1">' . + '<div class="center">' . + '<img src="/static/icon/' . $icon . '.png" alt="icon">' . + '<div class="title">' . $website . '</div>' . + '</div>' . + '</a>'; + } + + $right["answer"] .= '</div>'; + } + + $right["answer"] .= + '</div></div>' . + '<label class="spoiler-button" for="answer' . $answerlen . '"></label></div>'; + } +} + +/* + Add right containers +*/ +if(isset($right["answer"])){ + + if(count($right) >= 2){ + + $payload["right-right"] = $right["answer"]; + unset($right["answer"]); + } +} + +$c = 0; +foreach($right as $snippet){ + + if($c % 2 === 0){ + + $payload["right-left"] .= $snippet; + }else{ + + $payload["right-right"] .= $snippet; + } + + $c++; +} + +if($c !== 0){ + + $payload["class"] = " has-answer"; +} + +/* + Generate related searches +*/ +$c = count($results["related"]); + +if($c !== 0){ + $payload["left"] .= '<h3>Related searches</h3><table class="related">'; + + $opentr = false; + + for($i=0; $i<$c; $i++){ + + if(($i % 2) === 0){ + + $opentr = true; + $payload["left"] .= '<tr>'; + }else{ + + $opentr = false; + } + + $payload["left"] .= + '<td>' . + '<a href="/web?s=' . + urlencode($results["related"][$i]) . "&" . + $frontend->buildquery($get, true) . + '">' . + htmlspecialchars($results["related"][$i]) . + '</a>'; + + $payload["left"] .= '</td>'; + + if($opentr === false){ + + $payload["left"] .= '</tr>'; + } + } + + if($opentr === true){ + + $payload["left"] .= '<td></td></tr>'; + } + + $payload["left"] .= '</table>'; +} + +/* + Load next page +*/ +if($results["npt"] !== null){ + + $payload["left"] .= + '<a href="' . $frontend->htmlnextpage($get, $results["npt"], "web") . '" class="nextpage">Next page ></a>'; +} + +echo $frontend->load("search.html", $payload); |
