Link Scraper is a simple tool for scraping web page links. It returns all the links on a web page. To use this API, you need an API key. You can get one by creating a free account and visiting your dashboard.
Important: Ensure that this API is enabled from within your dashboard to use it in your application. If not, you may receive a 403 error
Scrape all the links on a web page
This is a required header on every request. Your API Key is found from within your dashboard
This is an optional header. Set the value to application/json, application/xml, or application/yaml
string(required)The URL of the web page to scrape links from
Sample Request
import requests
url = "https://api.apiverve.com/v1/linkscraper"
payload = { "url": "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html", "maxlinks": 20, "includequery": false }
headers = {
"x-api-key": "YOUR_API_KEY",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.json())
using System.Net.Http.Headers;
var client = new HttpClient();
var request = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri("https://api.apiverve.com/v1/linkscraper"),
Headers =
{
{ "x-api-key", "YOUR_API_KEY" }
},
Content = new StringContent("{ \"url\": \"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html\", \"maxlinks\": 20, \"includequery\": false }")
{
Headers =
{
ContentType = new MediaTypeHeaderValue("application/json")
}
}
};
using (var response = await client.SendAsync(request))
{
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
Console.WriteLine(body);
}
const data = JSON.stringify({ "url": "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html", "maxlinks": 20, "includequery": false });
const xhr = new XMLHttpRequest();
xhr.withCredentials = true;
xhr.addEventListener('readystatechange', function () {
if (this.readyState === this.DONE) {
console.log(this.responseText);
}
});
xhr.open('POST', 'https://api.apiverve.com/v1/linkscraper');
xhr.setRequestHeader('x-api-key', 'YOUR_API_KEY');
xhr.setRequestHeader('Content-Type', 'application/json');
xhr.setRequestHeader('Accept', 'application/json');
xhr.send(data);
const request = require('request');
const options = {
method: 'POST',
url: 'https://api.apiverve.com/v1/linkscraper',
headers: {
'x-api-key': 'YOUR_API_KEY',
'Content-Type': 'application/json',
Accept: 'application/json'
},
body: { "url": "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html", "maxlinks": 20, "includequery": false },
json: true
};
request(options, function (error, response, body) {
if (error) throw new Error(error);
console.log(body);
});
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create("https://api.apiverve.com/v1/linkscraper"))
.header("x-api-key", "YOUR_API_KEY")
.header("Content-Type", "application/json")
.header("Accept", "application/json")
.method("POST", HttpRequest.BodyPublishers.ofString("{ \"url\": \"https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html\", \"maxlinks\": 20, \"includequery\": false }"))
.build();
HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(response.body());
import Foundation
let headers = [
"x-api-key": "YOUR_API_KEY",
"Content-Type": "application/json"
]
let parameters = ["url" : "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html", "maxlinks" : 20, "includequery" : False,] as [String : Any]
let postData = JSONSerialization.data(withJSONObject: parameters, options: [])
let request = NSMutableURLRequest(url: NSURL(string: "https://api.apiverve.com/v1/linkscraper")! as URL,
cachePolicy: .useProtocolCachePolicy,
timeoutInterval: 10.0)
request.httpMethod = "POST"
request.allHTTPHeaderFields = headers
request.httpBody = postData as Data
let session = URLSession.shared
let dataTask = session.dataTask(with: request as URLRequest, completionHandler: { (data, response, error) -> Void in
if (error != nil) {
print(error as Any)
} else {
let httpResponse = response as? HTTPURLResponse
print(httpResponse)
}
})
dataTask.resume()
curl --request POST \
--url https://api.apiverve.com/v1/linkscraper \
--header 'Accept: application/json' \
--header 'Content-Type: application/json' \
--header 'x-api-key: YOUR_API_KEY' \
--data '{ "url": "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html", "maxlinks": 20, "includequery": false }'
Sample Response
{
"status": "ok",
"error": null,
"data": {
"url": "http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html",
"linkCount": 16,
"links": [
{
"text": "",
"href": "http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html/pdfs/AWSEC2/latest/UserGuide/ec2-ug.pdf#concepts",
"external": false
},
{
"text": "Documentation",
"href": "http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html/index.html",
"external": false
},
{
"text": "Amazon EC2",
"href": "http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html/ec2/index.html",
"external": false
},
{
"text": "User Guide",
"href": "concepts.html",
"external": true
},
{
"text": "Amazon EC2 Instance Types Guide",
"href": "https://docs.aws.amazon.com/ec2/latest/instancetypes/instance-types.html",
"external": true
},
{
"text": "PCI DSS Level 1",
"href": "https://aws.amazon.com/compliance/pci-dss-level-1-faqs/",
"external": true
},
{
"text": "Amazon EC2 Auto Scaling",
"href": "https://docs.aws.amazon.com/autoscaling/",
"external": true
},
{
"text": "AWS Backup",
"href": "https://docs.aws.amazon.com/aws-backup/",
"external": true
},
{
"text": "Amazon CloudWatch",
"href": "https://docs.aws.amazon.com/cloudwatch/",
"external": true
},
{
"text": "Elastic Load Balancing",
"href": "https://docs.aws.amazon.com/elasticloadbalancing/",
"external": true
},
{
"text": "Amazon GuardDuty",
"href": "https://docs.aws.amazon.com/guardduty/",
"external": true
},
{
"text": "EC2 Image Builder",
"href": "https://docs.aws.amazon.com/imagebuilder/",
"external": true
},
{
"text": "AWS Launch Wizard",
"href": "https://docs.aws.amazon.com/launchwizard/",
"external": true
},
{
"text": "AWS Systems Manager",
"href": "https://docs.aws.amazon.com/systems-manager/",
"external": true
},
{
"text": "Amazon Lightsail",
"href": "https://docs.aws.amazon.com/lightsail/",
"external": true
},
{
"text": "Amazon Lightsail or Amazon EC2",
"href": "https://docs.aws.amazon.com/decision-guides/latest/lightsail-or-ec2/lightsail-or-ec2.html",
"external": true
}
],
"maxLinksReached": false
},
"code": 200
}
<Root>
<status>ok</status>
<error />
<data>
<url>http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html</url>
<linkCount>16</linkCount>
<links>
<text></text>
<href>http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html/pdfs/AWSEC2/latest/UserGuide/ec2-ug.pdf#concepts</href>
<external>false</external>
</links>
<links>
<text>Documentation</text>
<href>http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html/index.html</href>
<external>false</external>
</links>
<links>
<text>Amazon EC2</text>
<href>http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html/ec2/index.html</href>
<external>false</external>
</links>
<links>
<text>User Guide</text>
<href>concepts.html</href>
<external>true</external>
</links>
<links>
<text>Amazon EC2 Instance Types Guide</text>
<href>https://docs.aws.amazon.com/ec2/latest/instancetypes/instance-types.html</href>
<external>true</external>
</links>
<links>
<text>PCI DSS Level 1</text>
<href>https://aws.amazon.com/compliance/pci-dss-level-1-faqs/</href>
<external>true</external>
</links>
<links>
<text>Amazon EC2 Auto Scaling</text>
<href>https://docs.aws.amazon.com/autoscaling/</href>
<external>true</external>
</links>
<links>
<text>AWS Backup</text>
<href>https://docs.aws.amazon.com/aws-backup/</href>
<external>true</external>
</links>
<links>
<text>Amazon CloudWatch</text>
<href>https://docs.aws.amazon.com/cloudwatch/</href>
<external>true</external>
</links>
<links>
<text>Elastic Load Balancing</text>
<href>https://docs.aws.amazon.com/elasticloadbalancing/</href>
<external>true</external>
</links>
<links>
<text>Amazon GuardDuty</text>
<href>https://docs.aws.amazon.com/guardduty/</href>
<external>true</external>
</links>
<links>
<text>EC2 Image Builder</text>
<href>https://docs.aws.amazon.com/imagebuilder/</href>
<external>true</external>
</links>
<links>
<text>AWS Launch Wizard</text>
<href>https://docs.aws.amazon.com/launchwizard/</href>
<external>true</external>
</links>
<links>
<text>AWS Systems Manager</text>
<href>https://docs.aws.amazon.com/systems-manager/</href>
<external>true</external>
</links>
<links>
<text>Amazon Lightsail</text>
<href>https://docs.aws.amazon.com/lightsail/</href>
<external>true</external>
</links>
<links>
<text>Amazon Lightsail or Amazon EC2</text>
<href>https://docs.aws.amazon.com/decision-guides/latest/lightsail-or-ec2/lightsail-or-ec2.html</href>
<external>true</external>
</links>
<maxLinksReached>false</maxLinksReached>
</data>
<code>200</code>
</Root>
status: ok
error:
data:
url: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html
linkCount: 16
links:
- text: ''
href: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html/pdfs/AWSEC2/latest/UserGuide/ec2-ug.pdf#concepts
external: false
- text: Documentation
href: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html/index.html
external: false
- text: Amazon EC2
href: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html/ec2/index.html
external: false
- text: User Guide
href: concepts.html
external: true
- text: Amazon EC2 Instance Types Guide
href: https://docs.aws.amazon.com/ec2/latest/instancetypes/instance-types.html
external: true
- text: PCI DSS Level 1
href: https://aws.amazon.com/compliance/pci-dss-level-1-faqs/
external: true
- text: Amazon EC2 Auto Scaling
href: https://docs.aws.amazon.com/autoscaling/
external: true
- text: AWS Backup
href: https://docs.aws.amazon.com/aws-backup/
external: true
- text: Amazon CloudWatch
href: https://docs.aws.amazon.com/cloudwatch/
external: true
- text: Elastic Load Balancing
href: https://docs.aws.amazon.com/elasticloadbalancing/
external: true
- text: Amazon GuardDuty
href: https://docs.aws.amazon.com/guardduty/
external: true
- text: EC2 Image Builder
href: https://docs.aws.amazon.com/imagebuilder/
external: true
- text: AWS Launch Wizard
href: https://docs.aws.amazon.com/launchwizard/
external: true
- text: AWS Systems Manager
href: https://docs.aws.amazon.com/systems-manager/
external: true
- text: Amazon Lightsail
href: https://docs.aws.amazon.com/lightsail/
external: true
- text: Amazon Lightsail or Amazon EC2
href: https://docs.aws.amazon.com/decision-guides/latest/lightsail-or-ec2/lightsail-or-ec2.html
external: true
maxLinksReached: false
code: 200
The Link Scraper API supports the following response content types:
application/json, application/xml, application/yaml
You can specify the response content type by setting the Accept header in your request. If you don't specify a content type, the API will default to application/json.
The Link Scraper API uses an API Key to authenticate requests. You can view and manage your API key by visiting your dashboard.
Your API keys carry many privileges. To keep them from being abused, please do not share the keys on client-side code or Github etc. Keep them very secure.
To use any API, you must have it enabled from within your dashboard. Disabled APIs will fail to respond to your requests.
All requests made to the API must contain the header x-api-key in each of your requests. API requests without authentication will fail.
All API requests must also be made over secure HTTPS. Requests made over plain HTTP will fail.
| Error Code | Meaning |
|---|---|
| 401 | Your request was made with invalid credentials. This error also appears when you don't pass the x-api-key header in your request.
|
| 403 | Typically, this occurs when you are trying to access an API that you have not enabled. |
{
"status": "error",
"data": null,
"error": "tokens have been depleted",
"code": 429
}
Each subscription has its own monthly token limit. Your token count is based on your subscription plan. If you reach your limits, don't worry. You can always upgrade or downgrade at any time.
When you reach your limit, the service will stop responding and typically return an HTTP 429 response status code. The error will also contain a detailed JSON.
The Link Scraper API uses the following error code:
| Error Code | Meaning |
|---|---|
| 429 | You have exceeded your rate limit and further requests will be denied until the next cycle. |
For reference, the Link Scraper API uses the following error codes:
| Error Code | Meaning |
|---|---|
| Code | Message |
| 200 | The request was successful. The response will include the requested data. |
| 400 | The request was invalid. The response will include a message that explains the error. |
| 401 | The request was not authorized. Usually, this means that the API key is missing or invalid. |
| 403 | This means that the request was trying to access a resource that it does not have permission to access. |
| 404 | This means that the resource you are trying to access does not exist. |
| 429 | This means that you have reached the rate limit. The response will include a Retry-After header that indicates how many seconds you need to wait before making a new request. |
| 500 | This means that there was an error on the server side. We are alerted when this happens and we will work to fix it as soon as possible. |