Meyar/Scraper/irandoc_jsonv2_scraper.py

import requests
import json
import time
import random

query = "اقتصاد مقاومتی"

session = requests.Session()
session.trust_env = False
session.proxies = {"http": None, "https": None}

headers = {
    "User-Agent": "Mozilla/5.0",
    "Accept": "application/json, text/plain, */*",
    "Referer": "https://ganj.irandoc.ac.ir/",
    "Origin": "https://ganj.irandoc.ac.ir"
}


# گرفتن abstract
def get_abstract(uid, retries=5):
    url = f"https://ganj.irandoc.ac.ir/api/v1/articles/{uid}/show_abstract"

    for i in range(retries):
        try:
            res = session.get(url, headers=headers, timeout=10)

            if res.status_code == 200:
                if not res.text.strip():
                    print(" empty → احتمال بلاک")
                else:
                    try:
                        data = res.json()
                        abstract = data.get("abstract")
                        if abstract:
                            return abstract
                    except:
                        pass

        except Exception as e:
            print("Error:", e)

        time.sleep(random.uniform(3, 7))

    return None


# گرفتن year
def get_year(uid, retries=5):
    url = f"https://ganj.irandoc.ac.ir/api/v1/articles/{uid}"

    for i in range(retries):
        try:
            res = session.get(url, headers=headers, timeout=10)

            if res.status_code == 200:
                try:
                    data = res.json()
                    return data.get("ja_pub_yyyy")
                except:
                    pass

        except Exception as e:
            print("Year Error:", e)

        time.sleep(random.uniform(3, 7))

    return None


page = 1
all_results = []
index_counter = 1


while True:
    print(f"\n Fetching page {page}...")

    search_url = (
        "https://ganj.irandoc.ac.ir/api/v1/search/main"
        f"?basicscope=1"
        f"&keywords={query}"
        f"&page={page}"
        f"&sort_by=1"
        f"&year_from=0"
        f"&year_to=1405"
    )

    response = session.get(search_url, headers=headers)

    if response.status_code != 200:
        print(" خطا در سرچ")
        break

    data = response.json()
    results = data.get("results", [])

    if not results:
        print(" پایان نتایج")
        break

    for item in results:
        title = item.get("title")
        uid = item.get("uuid")

        link = f"https://ganj.irandoc.ac.ir/articles/{uid}" if uid else None

        # چکیده + سال
        abstract = None
        year = None

        if uid:
            abstract = get_abstract(uid)
            year = get_year(uid)

        # نویسنده / استاد
        authors, advisors = [], []
        for c in item.get("contributions", []):
            role = c.get("role", {}).get("title_en", "").lower()
            name = c.get("researcher", {}).get("full_name_fa", "")
            if role == "author" and name:
                authors.append(name)
            elif role == "thesis advisor" and name:
                advisors.append(name)

        # دانشگاه
        universities = []
        for p in item.get("partnerships", []):
            org = p.get("organization", {})
            super_org = org.get("super_organization", {})
            uni = super_org.get("title") or org.get("title")
            if uni:
                universities.append(uni)

        # مقطع 
        grade = item.get("grade", {}).get("title_fa")
       

        all_results.append({
            "index": index_counter,
            "title": title,
            "year": year,
            "grade": grade,
            "authors": authors,
            "advisors": advisors,
            "universities": universities,
            "link": link,
            "abstract": abstract
        })

        index_counter += 1

        time.sleep(random.uniform(2, 5))

        if len(all_results) % 15 == 0:
            print("⏸ استراحت برای جلوگیری از بلاک...")
            time.sleep(20)

    page += 1
    time.sleep(random.uniform(2, 4))


final_output = {
    "query": query,
    "total_results": len(all_results),
    "results": all_results
}

with open("all_results.json", "w", encoding="utf-8") as f:
    json.dump(final_output, f, ensure_ascii=False, indent=2)

print("\n DONE")
print("Total:", len(all_results))