User-agent: *

# Disallow downloading archives or downloads
Disallow: /*/*/archive/
Disallow: /*/tarball/
Disallow: /*/zipball/
Disallow: /*/download/


Disallow: /api/*

Disallow: /*/*/compare/*
Disallow: /*/*/tree/*
Disallow: /*/*/blob/*
Disallow: /*/*/search
Disallow: /*/*/find/*
Disallow: /*/*/commit/*
Disallow: /*/*/*/refs/*
Disallow: /*/*/*/star
Disallow: /*/*/*/watch

Disallow: /user/login
Disallow: /user/sign_up

# Disallow general search and other url args
Disallow: /*?lang=
Disallow: /search
Disallow: /search/advanced
Disallow: /explore/*?*
Disallow: /*lang=*
Disallow: /*source=*
Disallow: /*ref_cta=*
Disallow: /*plan=*
Disallow: /*return_to=*
Disallow: /*ref_loc=*
Disallow: /*setup_organization=*
Disallow: /*source_repo=*
Disallow: /*ref_page=*
Disallow: /*referrer=*
Disallow: /*report=*
Disallow: /*author=*
Disallow: /*since=*
Disallow: /*until=*
Disallow: /*commits?author=*
Disallow: /*tab=*
Disallow: /*q=*
Disallow: /*repo-search-archived=*

# Disallow browsing files from commit ref
Disallow: /*/*/src/commit/*
# Disallow labels, graph or activity
Disallow: /*/*/labels
Disallow: /*/*/pulse
Disallow: /*/*/graph
Disallow: /*/*/activity
# Disallow file history from any ref
Disallow: /*/*/commits/commit/*
Disallow: /*/*/commits/branch/*
Disallow: /*/*/commits/tag/*
# Disallow blame from any ref
Disallow: /*/*/blame/commit/*
Disallow: /*/*/blame/branch/*
Disallow: /*/*/blame/tag/*

Crawl-delay: 10


# Annoying scrapers
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: GPTBot
User-agent: iaskspider/2.0
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
User-agent: PanguBot
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: Scrapy
User-agent: SemrushBot
User-agent: Sidetrade indexer bot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: SeznamBot
User-agent: AwarioBot
User-agent: AwarioSmartBot
User-agent: AwarioRssBot
User-agent: BLEXBot
Disallow: /


Sitemap: https://git.gammaspectra.live/sitemap.xml