init

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.12 (mlmodule1)" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (mlmodule1)" project-jdk-type="Python SDK" />
+</project>

.idea/mlmodule1.iml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.venv" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/mlmodule1.iml" filepath="$PROJECT_DIR$/.idea/mlmodule1.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

clearing_data.py

@@ -0,0 +1,70 @@
+import pandas as pd
+import ast
+import numpy as np
+import seaborn as sns
+import matplotlib.pyplot as plt
+data = pd.read_csv("raw_dataset.csv")
+columns = [
+    "experience",
+    "employment",
+    "salary_min",
+    "salary_max",
+]
+
+columns_for_cor = [
+    "experience",
+    "employment",
+    "salary_min",
+    "salary_max",
+    "area_Краснодар",
+    "area_Москва",
+    "area_Санкт-Петербург"
+]
+
+schedule = {
+    "Нет опыта": 0,
+    "От 1 года до 3 лет": 1,
+    "От 3 до 6 лет": 2,
+    "Более 6 лет": 3
+}
+
+employements = {
+
+    "Полная занятость": 0,
+    "Частичная занятость": 1,
+    "Проектная работа": 2,
+    "Волонтерство": 3,
+    "Стажировка": 4
+
+}
+
+data["area"] = data["area"].map(lambda city: ast.literal_eval(city)["name"])
+data["experience"] = data["experience"].map(lambda experience: ast.literal_eval(experience)["name"])
+data["employment"] = data["employment"].map(lambda employment: ast.literal_eval(employment)["name"])
+
+data["salary_min"] = data["salary"].map(
+    lambda salary_min: ast.literal_eval(salary_min)["from"]
+    if ast.literal_eval(salary_min)["from"] is not None
+    else ast.literal_eval(salary_min)["to"]
+)
+data["salary_max"] = data["salary"].map(
+    lambda salary_max: ast.literal_eval(salary_max)["to"]
+    if ast.literal_eval(salary_max)["to"] is not None
+    else ast.literal_eval(salary_max)["from"]
+)
+
+data["experience"] = data["experience"].map(lambda experience: schedule[experience])
+data["employment"] = data["employment"].map(lambda employment: employements[employment])
+
+data = data.groupby("area").filter(lambda values: len(values) > 30)
+
+data = pd.get_dummies(data, columns=["area"])
+
+
+
+
+data_cor = data[columns_for_cor].corr()
+
+sns.heatmap(data_cor)
+plt.show()
+print(data_cor)

extract.py

@@ -0,0 +1,43 @@
+import requests
+import pandas as pd
+
+
+params = {
+    "page": 0,
+    "per_page": 100,
+    "text": "bi",
+    "area": "113",
+    "currency": "RUR",
+    "date_from": "2024-07-22",
+    "only_with_salary": True
+}
+
+base_url = "https://api.hh.ru/vacancies"
+
+result = requests.get(base_url, params).json()
+
+vacancies = []
+vacancies.extend(result["items"])
+
+for i in range(1, result["pages"]):
+    params["page"] += 1
+    result = requests.get(base_url, params).json()
+    vacancies.extend(result["items"])
+
+data = pd.DataFrame.from_dict(vacancies)
+
+data.info()
+
+columns = [
+    "area",
+    "salary",
+    "schedule",
+    "url",
+    "experience",
+    "employment"
+]
+
+data = data[columns]
+
+
+data.to_csv("raw_dataset.csv")