import sys
def check_required_modules():
required_modules = ["requests", "xml.etree.ElementTree"]
missing_modules = []
for module in required_modules:
try:
__import__(module)
except ImportError:
missing_modules.append(module)
if missing_modules:
print("Missing required modules: " + ", ".join(missing_modules))
print("Please install them using pip. For example:")
print("pip install " + " ".join(missing_modules))
sys.exit(1)
# Place this check at the beginning of your main function
if __name__ == "__main__":
check_required_modules()
# Rest of your code follows...
import argparse
import json
import requests
import xml.etree.ElementTree as ET
import os
def fetch_sitemap_urls(sitemap_url):
try:
response = requests.get(sitemap_url)
if response.status_code == 200:
root = ET.fromstring(response.content)
return [loc.text.strip() for loc in root.findall('.//{http://www.sitemaps.org/schemas/sitemap/0.9}loc')]
else:
print(f"Failed to fetch sitemap: {sitemap_url}")
return []
except Exception as e:
print(f"An error occurred while fetching the sitemap: {str(e)}")
return []
def send_api_request(token, urls, starts_with, adaptive_type):
api_endpoint = "https://api.prerender.io/recache"
headers = {"Content-Type": "application/json"}
batch_size = 1000
url_batches = [urls[i:i + batch_size] for i in range(0, len(urls), batch_size)]
for i, url_batch in enumerate(url_batches, start=1):
payload = {
"prerenderToken": token,
"urls": url_batch,
"startsWith": starts_with,
"adaptiveType": adaptive_type
}
try:
response = requests.post(api_endpoint, data=json.dumps(payload), headers=headers)
if response.status_code == 200:
print(f"Batch {i}: Request successfully sent.")
print("Response:", response.text)
else:
print(f"Batch {i}: Request failed with status code: {response.status_code}")
print("Response:", response.text)
except Exception as e:
print(f"Batch {i}: An error occurred: {str(e)}")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Send requests to the Prerender API endpoint in batches")
parser.add_argument("token", help="Your prerender token")
parser.add_argument("--input_file", help="Path to the text file containing URLs")
parser.add_argument("--starts_with", default="string", help="String to include in the request (default: 'string')")
parser.add_argument("--sitemap", help="URL of the sitemap to fetch URLs from")
parser.add_argument("--adaptive_type", default="desktop", choices=['desktop', 'mobile'], help="Type of caching (default: 'desktop')")
args = parser.parse_args()
urls = []
if args.sitemap:
urls.extend(fetch_sitemap_urls(args.sitemap))
if args.input_file:
absolute_file_path = os.path.abspath(args.input_file)
with open(absolute_file_path, "r") as file:
file_urls = [line.strip() for line in file.readlines()]
urls.extend(file_urls)
if not urls:
print("No URLs provided. Please provide a sitemap URL or an input file.")
sys.exit(1)
send_api_request(args.token, urls, args.starts_with, args.adaptive_type)