 
 


         .           -,                .      ,         ,  , , ,   .                .       .    Big Data      .       ,      .   , ,   ,        -    .





 

 




  

1.   

2.    

3.    

4.    

5.    

6.    

7. SVM    

8. RF   

9. CNN    

10. RNN    

11. MLP   

12. SGD    

13. NLP    

14. CV   

15. DL   

16. ROI   

17. KPI    

18. CRM     

19. ERP     

20. BI  -






 1:         



1.1.       

      ,            .

     ,   .

 ( ,  )  ,          .

   ,         .

  ,             .

        ,    ,      .

   ,           ,       .

   ,            ,        .

   ,             , ,  ,         .

     ,        , ,  ,    (SVM),  ,    .

  ,              .

-     ,          ()     ,    .

    ,      , , , , F-,   (MSE)  .

         ,  , ,    ,     .

     ,    .

        ,  ,   , ,   ,     .

     ,     ,    ,    .

   ,            , , ,     .

    ,   ,    ,          .

     ,         .       ,    .

    ,         .        .

    ,         ,     .          .

    ,           . ,          .

    ,          .   , , ,        .

     ,     ,   .           .      ,       .

       ,         .                   .

      ,        ,     .           .

       ,      ,      .      ,       .

              .

                .

    ,           .            .

   ,             .   k-,    DBSCAN.

    ,                 .

   (PCA)            ,         .

   ,           .      ,      .

   ,     ,     .      (GAN)   .

             .


1.2.        

               .    ,                 .            .

1.    

                 .        ,      ,       .

               .         ,  ,     ,       .         ,    ,     -.

           .        ,  ,  ,     ,     ,      .        ,          .

           .        ,  ,    ,       .        ,      .

   ,          .              ,       .        ,       .

2.    -

         ,      .       ,          .

  ,        ,   . -,     ,            .       ,       .      ,            .

         .        -  ,         .           -       .

 -             . ,            ,           .           ,       .

           .      ,            .

3.     

           .              ,        .

 ,   ,      , ,    .               . ,         ,     ,     .

     ,      .    ,      ,      .       ,      .

 ,      ,     . ,        ,  ,     ,      .         ,          .

            .     ,      .

4.     

             .          ,     ,       .

       ,        .        ,   ,        .

           ,               . ,          ,            .         , ,          .

       ,        .  ,         ,   ,    ,      .

,  ,               .        ,        ,    .  ,      ,             .

              .       ,     ,         .

           .      ,          . ,                   .

5.    

       ,         .          ,     ,      .

             . ,         ,  ,            .           .

 ,       ,  -    . ,       ,   ,          .    ,       .

 ,             .         ,     ,         .               .

 ,        ,   .           ,       .         ,  -     .

 ,        .    ,  -,    ,        .             .


1.3.        

     ,       .

   ,         ,   .             .

            ,      .       ,  ,    .         ,         .

     .      ,         . ,            ,           .

        ,         .         . ,          ,  ,    ,    .

    ,      ,    .     ,    ,    ,      .

,    ,        .               .      ,  ,   ,     ,       .

 ,       ,   .   ,    ,       .  ,       ,        .

      ,     ,     ,       .         ,   ,       .

         . -         ,     .   ,      ,     ,  .

        ,       " "   .      ,       ,   " "      .

,            .          ,  ,           .

       .           ,           .

  ,        ,     .          ,   .

,      ,         .   ,       , ,    ,           .

  ,      .               .        ,      .

                 .      ,          .

   ,    ,         ,       .                 .

,   ,             .      ,       ,            .

     ,        .        .             .      ,    ,     .      ,  ,           .

 ,       .   ,  ,    ,    . ,     ,     ,            .            ,        .

       .          .       ,            .     , ,      .        ,           .

      ,        ,      .        ,   ,         ,           .

     ,              .                      .






 2:      



2.1.   

          .                .        , ,     .            ,       .

,   ,               .            . ,                    .

        .          ,   ,      . ,     ,              .          ,      .

            . ,               ,      .

                 . ,      ,     ,   , , ,  ,     .

    , ,  ,      :   .      ,     ,     . ,   ,            .

            . ,         ,              .         .

  ,                ,         .

    Python,   scikit-learn       :

```python

#   

from sklearn.model_selection import train_test_split

from sklearn.linear_model import LogisticRegression

from sklearn.metrics import accuracy_score

#   

# ,     CSV-     

#  : , , ,  ,      (/)

data = pd.read_csv("bank_clients.csv")

#     (X)    (y)

X = data.drop("target", axis=1)

y = data["target"]

#       

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

#    

model = LogisticRegression()

#      

model.fit(X_train, y_train)

#      

y_pred = model.predict(X_test)

#   

accuracy = accuracy_score(y_test, y_pred)

print(" : {:.2f}".format(accuracy))

```

               .     CSV-,       ,          .       ,         . ,        accuracy_score.

 ,                scikit-learn  pandas  .

  (Logistic Regression)         .          .

      ,     (    )           .      :

p(y=1|x) = sigmoid(w^T * x + b)

:

p(y=1|x)        1     x,

w   ,  ,

b   (bias),

sigmoid   ,   sigmoid(z) = 1 / (1 + exp(-z)).

        ,         ,     .

    ,               1.           .

            ,  , ,   .     .

-,        .      ,       .              .                  .

-,       .               .      ,    .

-,        ,      .      ,       ,       .


2.2.   

        ,                .        ,  ,  , ,     .

        ()  ,          ,      .        ,      . ,             ,          .

                .              . ,               ,         .

       ,             .      ,     ,    ,  ,     .

            K- (K-means)    Python:

```python

#   

import pandas as pd

import numpy as np

from sklearn.cluster import KMeans

import matplotlib.pyplot as plt

#  

data = pd.read_csv("bank_data.csv") # ,         

#  

X = data[['Age', 'Income']] #  ,     

#  

from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()

X_scaled = scaler.fit_transform(X)

#    

inertia = []

for k in range(1, 10):

kmeans = KMeans(n_clusters=k, random_state=42)

kmeans.fit(X_scaled)

inertia.append(kmeans.inertia_)

#   

plt.plot(range(1, 10), inertia, marker='o')

plt.xlabel('Number of clusters')

plt.ylabel('Inertia')

plt.title('Elbow Method')

plt.show()

#    

k = 3 #    ,      3

#   K-

kmeans = KMeans(n_clusters=k, random_state=42)

kmeans.fit(X_scaled)

#     

data['Cluster'] = kmeans.labels_

#  

for cluster in range(k):

cluster_data = data[data['Cluster'] == cluster]

print(f"Cluster {cluster + 1}:\n{cluster_data.describe()}\n")

```

 :

1.   ,   pandas    , numpy   , sklearn    K-  matplotlib  .

2.     "bank_data.csv". ,          ,   (Age),  (Income)   .

3.   (Age  Income)       DataFrame X.

4.         StandardScaler.

5.         (Elbow Method)   .

6.     (    3).

7.   K-    .

8.      .

9.      .

:    ,      "bank_data.csv"     .

 K- (K-means)           .          .

   K-   :

1.    (K):   ,    .               .

2.  :       ,         .      K.

3.    :            ,     .

4.  :        .           .

5.   3  4:            ,       .         .

6.  :      ,        .

,    K-     ,   :

d(x, c) = sqrt((x1  c1)^2 + (x2  c2)^2 +  + (xn  cn)^2)

:

d(x, c)       x   c,

x1, x2, , xn     x,

c1, c2, , cn    c.

           .            ,           .

   ,        .     ,    ,       .

 K-             ,              .

 K-   ,           .       ,           ,  , ,   .

           K- (K-means).            ,             .

```python

import pandas as pd

from sklearn.cluster import KMeans

from sklearn.preprocessing import StandardScaler

#     

data = pd.read_csv('customer_data.csv')

#  :   

scaler = StandardScaler()

scaled_data = scaler.fit_transform(data[['Age', 'Income', 'Balance']])

#   

k = 3

#     K-

kmeans = KMeans(n_clusters=k, random_state=42)

kmeans.fit(scaled_data)

#      

cluster_labels = kmeans.labels_

#      

data['Cluster'] = cluster_labels

#   

for cluster in range(k):

cluster_data = data[data['Cluster'] == cluster]

print(f"Cluster {cluster}:")

print(cluster_data.describe())

print('\n')

#   :

#     ,      

#          

```

      pandas  scikit-learn      ,      K-.       StandardScaler,       .

     (   k = 3)     KMeans.     fit,     ,           .

      .     ,    ,        .            .

  ,      ,       ,          .


2.3.   

            .      ,              .      ,       .

        .      ,         .      ,     .               .

 ,       ,   .            .            .

       ,         .      ,         .                        .

        . ,                 .        ,      .                .

                .              .    ,      .

1.   (Mean Squared Error, MSE):          .               .    MSE,       .    MSE:

MSE = (1/n) * ?(y  y)?,

 n   , y     , y     .

2.   (R-squared):   ,     .       ,   .        0  1,  0 ,      ,  1 ,     .     :

R? = 1  (SSR / SST),

 SSR    , SST       .

3.    (Mean Absolute Error, MAE):               .      ,   .    MAE:

MAE = (1/n) * ?|y  y|.

4.     (Root Mean Squared Error, RMSE):  

                  . RMSE      ,    ,   .    RMSE:

RMSE = ?MSE.

   ,        ,    ,    ,            .

           ,         .

              :

1.  : ,               . ,      ,    (MSE)      (RMSE)    .         ,    (R-squared)    .

2.  :    ,    ,     .  ,     (MSE),     ,        (MAE)    .  ,            , ,    .

3. -:       .      ? ,      ,       .      ,     (MAE)         .

4.  :      ,    , ,        .            .

 ,                 .            .

          .            .        ,       .


2.4.  

       ,      , ,    .      ,                .

            .      ,         ,       .        ,            .

       ,   , ,    .   , ,     ,       ,             ,     .

 

    ,    ,   .     ,         ,         .       (,    )            ,    .

 ,      :

```python

import numpy as np

#    

ratings = np.array([

[5, 4, 0, 0, 0, 0],

[0, 0, 4, 0, 5, 0],

[0, 0, 0, 2, 4, 5],

[4, 0, 0, 0, 0, 4]

])

#       

def compute_similarity(user1, user2):

mask = np.logical_and(user1 != 0, user2 != 0)

if np.sum(mask) == 0:

return 0

return np.corrcoef(user1[mask], user2[mask])[0, 1]

#     

def recommend_movies(user_id, ratings, num_recommendations=5):

num_users, num_movies = ratings.shape

#      

similarities = []

for i in range(num_users):

if i != user_id:

similarity = compute_similarity(ratings[user_id], ratings[i])

similarities.append((i, similarity))

similarities.sort(key=lambda x: x[1], reverse=True)

#  -N   

top_similar_users = [similarity[0] for similarity in similarities[:num_recommendations]]

#       

recommendations = np.zeros(num_movies)

for user in top_similar_users:

recommendations += ratings[user]

recommendations = np.where(ratings[user_id] == 0, recommendations, 0)

top_movies = np.argsort(recommendations)[::-1][:num_recommendations]

return top_movies

#  

user_id = 0

recommended_movies = recommend_movies(user_id, ratings)

print(f"    {user_id}:")

for movie_id in recommended_movies:

print(f" {movie_id}")

```

       `ratings`,     ,     .       0  5,  0   .

 `compute_similarity`       .     ,   ,    .

 `recommend_movies`          .       ,  -N          .




  ,       .      ,      .

,         .               .

         Singular Value Decomposition (SVD)   :

import numpy as np

from scipy.sparse import csr_matrix

from scipy.sparse.linalg import svds

#    

ratings = np.array([

[5.0, 4.0, 0.0, 0.0, 0.0, 0.0],

[0.0, 0.0, 4.0, 0.0, 5.0, 0.0],

[0.0, 0.0, 0.0, 2.0, 4.0, 5.0],

[4.0, 0.0, 0.0, 0.0, 0.0, 4.0]

])

#    (SVD)

def perform_svd(ratings, k):

#      

sparse_ratings = csr_matrix(ratings)

#  SVD    U, Sigma  Vt

U, Sigma, Vt = svds(sparse_ratings, k)

#    Sigma

Sigma = np.diag(Sigma)

return U, Sigma, Vt

#     

def recommend_movies(user_id, ratings, U, Sigma, Vt, num_recommendations=5):

user_ratings = ratings[user_id]

predicted_ratings = np.dot(np.dot(U[user_id, :], Sigma), Vt)

#      

predicted_ratings[user_ratings != 0] = -1

top_movies = np.argsort(predicted_ratings)[::-1][:num_recommendations]

return top_movies

#  

user_id = 0

k = 2 #   

U, Sigma, Vt = perform_svd(ratings, k)

recommended_movies = recommend_movies(user_id, ratings, U, Sigma, Vt)

print(f"    {user_id}:")

for movie_id in recommended_movies:

print(f" {movie_id}")

```

     Singular Value Decomposition (SVD)       .   U, Sigma  Vt            .

 `perform_svd`         `svds`   `scipy.sparse.linalg`.    U, Sigma  Vt.

 `recommend_movies`   ,  ,    U, Sigma  Vt   .         ,    ,    .

           0.     `num_recommendations`.




Singular Value Decomposition (SVD),   ,     ,     ,   ,  ,     .

         : U, Sigma  Vt. ,   A  m x n SVD   :

A = U * Sigma * Vt,

 U    m x m,    ,

Sigma     m x n,   ,

Vt     n x n,    .

    Sigma       .                  A.

  SVD   , ,  A     ,    ,     (,   ..). SVD     ,   ,       .     ,    ,        .

 SVD   ,           .     Truncated SVD (SVD     ), Implicit Matrix Factorization (IMF)  .

SVD           ,       .   ,  SVD              .         SVD   ,     .

, SVD -          ,      .

  

      .       ,         .    ,   (, ,   ..)     ,      .

            .      ,  , ,  ,    ,          .

,    ,   ,      .            , ,     .

,    ,        .           .

   ,      ,     . ,      ,       .

    ,         ,            .       ,         .

,      .  ,        ,         .            .

     ,              .        ,   , -,   ,             .

      ,          . ,               ,        .

        ,     .      ,      ,       -.

     :

```python

#   

import pandas as pd

from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.metrics.pairwise import cosine_similarity

#  

data = pd.read_csv('movies.csv')

#   TF-IDF    

tfidf = TfidfVectorizer(stop_words='english')

tfidf_matrix = tfidf.fit_transform(data['description'].fillna(''))

#     

cosine_sim = cosine_similarity(tfidf_matrix, tfidf_matrix)

#      

def get_recommendations(title, cosine_sim, data, top_n=5):

indices = pd.Series(data.index, index=data['title']).drop_duplicates()

idx = indices[title]

sim_scores = list(enumerate(cosine_sim[idx]))

sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)

sim_scores = sim_scores[1:top_n+1]

movie_indices = [i[0] for i in sim_scores]

return data['title'].iloc[movie_indices]

#      

movie_title = 'The Dark Knight Rises'

recommendations = get_recommendations(movie_title, cosine_sim, data)

print(f"   '{movie_title}':")

print(recommendations)

```

   :

1.   .  `pandas`        ,   `TfidfVectorizer`  `cosine_similarity`   `sklearn.feature_extraction.text`  `sklearn.metrics.pairwise`            .

2.       'movies.csv'   `read_csv()`   `pandas`.      ,   ,    .

3.   `TfidfVectorizer`,            TF-IDF. TF-IDF (Term Frequency-Inverse Document Frequency)    ,       .        .

4.    `fit_transform()`  `TfidfVectorizer`       TF-IDF.        ,      ,      .

5.         `cosine_similarity()`   `sklearn.metrics.pairwise`.            .   ,             .

6.   `get_recommendations()`,    ,      .    :

  `pd.Series`  ,     ,     .

   .

       .

    .

 -N    .

   .

7.     ,     .

8.   `get_recommendations()`     ,      .

9.     .

       TF-IDF           .          TF-IDF      .         .     ,        ,   .




 3:     


    .           .


           .          ,   .         ,    ,  ,     .

  ,      ,       .           ,         .     ,    ,      .

       -.  ,    ,       .    -       . ,       ,          .

          .                  .             .

      ,    ,   ,     ,      .      ,            .


3.1. ,    

       ,       .            .     ,                 .

       .            .      , , ,    ,        .      ,  , ,  , API   .    ,       ,         .

       .    , ,    .      .          ,   ,        .    ,    ,       .

      .      , ,      . ,     ,       ,          One-Hot Encoding      .            ,       .

 ,  ,       .             ,      .      ,           .                   .


3.1.1.  

           .          .

              .        ,       , ,   , ,     .  ,               .

    ,     .       ,  ,    ,      (IoT),       API (Application Programming Interface).         .

        .  ,     , ,     .           ,     .                 .

        . ,       -,     .         SQL-      . SQL (Structured Query Language)         .   SQL-  ,       ,      .

      IoT (Internet of Things)          .       ,   , ,   .            ,   Bluetooth, Wi-Fi     .

 API (Application Programming Interface)        . API     ,        .   API     ,  ,     .             .

              . ,   SQL-      SQL    .      IoT-     ,         .  API      ,          .

        ,      .       ,        .

         .  ,   ,       ,      .        .         ?           ?   ,    ,      ,     .

     ,    .           ?      ,     . ,         ,       .

     ,       .     ,    API,   ,     . ,         ,  ,         .

            .        ,    ,      . ,                     .

    ,    .         ,        .              .

           .             .               .

SQL-              .      :

1. SQL-: SQL (Structured Query Language)         .   SQL-    ,      , , ,    . SQL            .          ,    ,     .

2.     :   ,           .         ,        .        MySQL Workbench, Microsoft SQL Server Management Studio, Oracle SQL Developer  .       SQL-,    ,     .

   SQL-         . SQL-      ,          .   ,         ,        .     ,     SQL        .

 ,    SQL-           .       ,     ,        .

       ,            .

    SQL-         :

1.   SQL-:

,              -.    SQL-     ,     :

```sql

SELECT * FROM Customers

JOIN Orders ON Customers.CustomerID = Orders.CustomerID

WHERE Orders.OrderDate = '2023-05-31';

```

            ,  31  2023 .

2.    :

,          .     MySQL Workbench     .            , ,       :

 MySQL Workbench     .

   (, "Employees").

  "Execute SQL"  :

```sql

SELECT * FROM Employees WHERE Department = 'Marketing';

```

  "Execute"  "Run"   .

      ,    .

3.   SQL-:

,            .    SQL-       :

```sql

SELECT Subject, AVG(Grade) AS AverageGrade

FROM Students

GROUP BY Subject;

```

               .




  .


   .

   ,     (https://www.litres.ru/book/dzheyd-karter/mashinnoe-obuchenie-69356998/)  .

      Visa, MasterCard, Maestro,    ,   ,     ,  PayPal, WebMoney, ., QIWI ,       .


