Commit a6fadf41 authored by boncho vylkov's avatar boncho vylkov

add crawler, and some products and view for displaying the products

parent b7d40f9a
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp1.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AngleSharp" Version="0.9.9" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\MvcCoreTemplate.Data\MvcCoreTemplate.Data.csproj" />
<ProjectReference Include="..\MvcCoreTemplate.Services.Data\MvcCoreTemplate.Services.Data.csproj" />
</ItemGroup>
</Project>
\ No newline at end of file
using AngleSharp;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using MvcCoreTemplate.Data;
using MvcCoreTemplate.Data.Models;
using MvcCoreTemplate.Data.UnitOfWork;
using MvcCoreTemplate.Services.Data;
using System;
using System.Linq;
using System.Threading;
namespace MvcCoreTemplate.Crawler
{
class Program
{
static void Main(string[] args)
{
//setup our DI
var serviceProvider = new ServiceCollection()
.AddDbContext<ApplicationDbContext>(options =>
{
options.UseSqlServer("Server=.\\SQLExpress;Database=MvcCoreTemplateDb;Trusted_Connection=True;MultipleActiveResultSets=true");
})
.AddScoped(typeof(IUowData), typeof(UowData))
.AddTransient<IProductService, ProductService>()
.BuildServiceProvider();
var productSetvice = serviceProvider.GetService<IProductService>();
Console.WriteLine(productSetvice);
var configuration = Configuration.Default.WithDefaultLoader();
var browsingContext = BrowsingContext.New(configuration);
var baseUrl = "https://www.heb.com";
var categoriesUrl = baseUrl + "/category/shop/food-and-drinks/bakery/cakes/2951";
var document = browsingContext.OpenAsync(categoriesUrl).Result;
var categoriesContainer = document.QuerySelector("ul.cat-list");
//.TextContent.Trim();
var allCategories = categoriesContainer.QuerySelectorAll("li");
foreach (var cat in allCategories)
{
var catElement = cat.QuerySelector("div.cat-list-deparment a");
var catName = catElement.TextContent.Trim().Split(new char[] { '('}).FirstOrDefault();
var catUrl = catElement.GetAttribute("href");
var catImageUrl = cat.QuerySelector("div.cat-list-img a img").GetAttribute("src");
Console.WriteLine(catUrl);
if (!productSetvice.CategoryExist(catName))
{
Category newCat = new Category();
newCat.ImageUrl = catImageUrl;
newCat.Name = catName;
int catId = productSetvice.AddProductCategory(newCat);
var categoryDocument = browsingContext.OpenAsync(baseUrl + catUrl).Result;
var productsContainer = categoryDocument.QuerySelector("ul.cat-list");
var allProducts = productsContainer.QuerySelectorAll("li.gridproductview");
foreach (var pr in allProducts)
{
var productImage = pr.QuerySelector("div.cat-list-img img").GetAttribute("src");
var productName = pr.QuerySelector("div.cat-list-deparment a").TextContent.Trim();
var productPrice = pr.QuerySelector("div.cat-price span").TextContent.Trim()
.TrimStart(new char[] { '$' }).Split(new char[] { ' ' }).FirstOrDefault();
Product newProduct = new Product()
{
CategoryId = catId,
Name = productName,
Price = decimal.Parse(productPrice),
ImageUrl = productImage,
IsInHouse = true,
};
productSetvice.AddProduct(newProduct);
}
Thread.Sleep(500);
}
}
//for (int i = 1; i <= 10000; i++)
//{
// var url = $"http://vicove.com/vic-{i}";
// var document = browsingContext.OpenAsync(url).Result;
// var jokeContent = document.QuerySelector("#content_box .post-content").TextContent.Trim();
// if (!string.IsNullOrWhiteSpace(jokeContent))
// {
// var categoryName = document.QuerySelector("#content_box .thecategory a").TextContent.Trim();
// var category = categoriesService.EnsureCategory(categoryName);
// var joke = new Joke() { Category = category, Content = jokeContent };
// db.Jokes.Add(joke);
// db.SaveChanges();
// Console.WriteLine(i);
// }
//}
}
}
}
\ No newline at end of file
......@@ -7,5 +7,8 @@ namespace MvcCoreTemplate.Data.Models
{
[MaxLength(256)]
public string Name { get; set; }
[MaxLength(256)]
public string ImageUrl { get; set; }
}
}
\ No newline at end of file
......@@ -9,10 +9,10 @@ namespace MvcCoreTemplate.Data.Models
public PlacedOrder()
{
this.Products = new HashSet<Product>();
this.ProductPlacedOrders = new HashSet<ProductPlacedOrder>();
}
public virtual ICollection<Product> Products { get; set; }
public ICollection<ProductPlacedOrder> ProductPlacedOrders { get; set; }
//all these would be filled automatically if the user is registered
//public int OrderDetailsId { get; set; }
......
......@@ -9,6 +9,7 @@ namespace MvcCoreTemplate.Data.Models
public Product()
{
this.GalleryImages = new HashSet<GalleryImage>();
this.ProductPlacedOrders = new HashSet<ProductPlacedOrder>();
}
[MaxLength(512)]
......@@ -31,7 +32,9 @@ namespace MvcCoreTemplate.Data.Models
[Required]
public string ImageUrl { get; set; }
public virtual ICollection<GalleryImage> GalleryImages { get; set; }
public ICollection<GalleryImage> GalleryImages { get; set; }
public ICollection<ProductPlacedOrder> ProductPlacedOrders { get; set; }
}
}
using System;
using System.Collections.Generic;
using System.Text;
namespace MvcCoreTemplate.Data.Models
{
public class ProductPlacedOrder
{
public int ProductId { get; set; }
public Product Product { get; set; }
public int PlacedOrderId { get; set; }
public PlacedOrder PlacedOrder { get; set; }
}
}
......@@ -18,6 +18,21 @@ namespace MvcCoreTemplate.Data
protected override void OnModelCreating(ModelBuilder builder)
{
builder.Entity<ProductPlacedOrder>()
.HasKey(bc => new { bc.ProductId, bc.PlacedOrderId });
builder.Entity<ProductPlacedOrder>()
.HasOne(bc => bc.Product)
.WithMany(b => b.ProductPlacedOrders)
.HasForeignKey(bc => bc.ProductId);
builder.Entity<ProductPlacedOrder>()
.HasOne(bc => bc.PlacedOrder)
.WithMany(c => c.ProductPlacedOrders)
.HasForeignKey(bc => bc.PlacedOrderId);
base.OnModelCreating(builder);
// Customize the ASP.NET Identity model and override the defaults if needed.
// For example, you can rename the ASP.NET Identity table names and more.
......@@ -44,6 +59,9 @@ namespace MvcCoreTemplate.Data
public DbSet<PlacedOrder> PlacedOrders { get; set; }
public override int SaveChanges()
{
this.ApplyAuditInfoRules();
......
using System;
using System.Collections.Generic;
using Microsoft.EntityFrameworkCore.Migrations;
namespace MvcCoreTemplate.Data.Migrations
{
public partial class UpdateProductCategory : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "ImageUrl",
table: "Categories",
maxLength: 256,
nullable: true);
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "ImageUrl",
table: "Categories");
}
}
}
using System;
using System.Collections.Generic;
using Microsoft.EntityFrameworkCore.Migrations;
namespace MvcCoreTemplate.Data.Migrations
{
public partial class UpdateProduct : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_Products_PlacedOrders_PlacedOrderId",
table: "Products");
migrationBuilder.DropIndex(
name: "IX_Products_PlacedOrderId",
table: "Products");
migrationBuilder.DropColumn(
name: "PlacedOrderId",
table: "Products");
migrationBuilder.CreateTable(
name: "ProductPlacedOrder",
columns: table => new
{
ProductId = table.Column<int>(nullable: false),
PlacedOrderId = table.Column<int>(nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_ProductPlacedOrder", x => new { x.ProductId, x.PlacedOrderId });
table.ForeignKey(
name: "FK_ProductPlacedOrder_PlacedOrders_PlacedOrderId",
column: x => x.PlacedOrderId,
principalTable: "PlacedOrders",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
table.ForeignKey(
name: "FK_ProductPlacedOrder_Products_ProductId",
column: x => x.ProductId,
principalTable: "Products",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "IX_ProductPlacedOrder_PlacedOrderId",
table: "ProductPlacedOrder",
column: "PlacedOrderId");
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "ProductPlacedOrder");
migrationBuilder.AddColumn<int>(
name: "PlacedOrderId",
table: "Products",
nullable: true);
migrationBuilder.CreateIndex(
name: "IX_Products_PlacedOrderId",
table: "Products",
column: "PlacedOrderId");
migrationBuilder.AddForeignKey(
name: "FK_Products_PlacedOrders_PlacedOrderId",
table: "Products",
column: "PlacedOrderId",
principalTable: "PlacedOrders",
principalColumn: "Id",
onDelete: ReferentialAction.Restrict);
}
}
}
......@@ -224,6 +224,9 @@ namespace MvcCoreTemplate.Data.Migrations
b.Property<DateTime?>("DeletedOn");
b.Property<string>("ImageUrl")
.HasMaxLength(256);
b.Property<bool>("IsDeleted");
b.Property<DateTime?>("ModifiedOn");
......@@ -384,8 +387,6 @@ namespace MvcCoreTemplate.Data.Migrations
b.Property<string>("Name")
.HasMaxLength(512);
b.Property<int?>("PlacedOrderId");
b.Property<decimal>("Price");
b.HasKey("Id");
......@@ -394,8 +395,6 @@ namespace MvcCoreTemplate.Data.Migrations
b.HasIndex("CategoryId");
b.HasIndex("PlacedOrderId");
b.ToTable("Products");
});
......@@ -425,6 +424,19 @@ namespace MvcCoreTemplate.Data.Migrations
b.ToTable("ProductOrders");
});
modelBuilder.Entity("MvcCoreTemplate.Data.Models.ProductPlacedOrder", b =>
{
b.Property<int>("ProductId");
b.Property<int>("PlacedOrderId");
b.HasKey("ProductId", "PlacedOrderId");
b.HasIndex("PlacedOrderId");
b.ToTable("ProductPlacedOrder");
});
modelBuilder.Entity("MvcCoreTemplate.Data.Models.Setting", b =>
{
b.Property<int>("Id")
......@@ -534,10 +546,6 @@ namespace MvcCoreTemplate.Data.Migrations
b.HasOne("MvcCoreTemplate.Data.Models.Category", "Category")
.WithMany()
.HasForeignKey("CategoryId");
b.HasOne("MvcCoreTemplate.Data.Models.PlacedOrder")
.WithMany("Products")
.HasForeignKey("PlacedOrderId");
});
modelBuilder.Entity("MvcCoreTemplate.Data.Models.ProductOrder", b =>
......@@ -552,6 +560,19 @@ namespace MvcCoreTemplate.Data.Migrations
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("MvcCoreTemplate.Data.Models.ProductPlacedOrder", b =>
{
b.HasOne("MvcCoreTemplate.Data.Models.PlacedOrder", "PlacedOrder")
.WithMany("ProductPlacedOrders")
.HasForeignKey("PlacedOrderId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("MvcCoreTemplate.Data.Models.Product", "Product")
.WithMany("ProductPlacedOrders")
.HasForeignKey("ProductId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("MvcCoreTemplate.Data.Models.GalleryImage", b =>
{
b.HasOne("MvcCoreTemplate.Data.Models.Product", "Product")
......
......@@ -22,6 +22,8 @@ namespace MvcCoreTemplate.Data.UnitOfWork
Task<int> SaveChangesAsync();
int SaveChanges();
void Dispose();
}
}
......@@ -69,5 +69,10 @@ namespace MvcCoreTemplate.Data.UnitOfWork
{
return await context.SaveChangesAsync();
}
public int SaveChanges()
{
return context.SaveChanges();
}
}
}
namespace MvcCoreTemplate.Services.Data
{
using MvcCoreTemplate.Data.Models;
using MvcCoreTemplate.Web.Infrastructure.Models.Products;
using System.Collections.Generic;
public interface IProductService
{
IEnumerable<ProductListViewModel> GetProductsByName(string name, int? skip = 0, int? take = 10);
IEnumerable<ProductListViewModel> GetProductsByName(string name, int? skip = 0, int? take = 12);
IEnumerable<ProductListViewModel> GetProductsByCategory(int categoryId, int? skip = 0, int? take = 10);
IEnumerable<ProductListViewModel> GetProductsByCategory(int categoryId, int? skip = 0, int? take = 12);
IEnumerable<ProductListViewModel> GetProductsByCategoryName(string categoryName, int? skip = 0, int? take = 10);
IEnumerable<ProductListViewModel> GetProductsByCategoryName(string categoryName, int? skip = 0, int? take = 12);
IEnumerable<ProductListViewModel> GetProductsByNameAndToPrice(string name,decimal price, int? skip = 0, int? take = 10);
IEnumerable<ProductListViewModel> GetProductsByNameAndToPrice(string name,decimal price, int? skip = 0, int? take = 12);
IEnumerable<ProductListViewModel> GetProductsByNameAndFromPrice(string name, decimal price, int? skip = 0, int? take = 10);
IEnumerable<ProductListViewModel> GetLatestProducts(int? skip = 0, int? take = 10);
IEnumerable<ProductListViewModel> GetProductsByNameAndFromPrice(string name, decimal price, int? skip = 0, int? take = 12);
int GetPagesCount(int? skip, int? take);
IEnumerable<ProductListViewModel> GetLatestProducts(int? skip = 0, int? take = 12);
ProductDetailsViewModel GetProductDetails(int id);
int AddProductCategory(Category category);
bool CategoryExist(string name);
int AddProduct(Product product);
}
}
......@@ -18,6 +18,25 @@ namespace MvcCoreTemplate.Services.Data
this.data = data;
}
public int AddProduct(Product product)
{
this.data.Products.Add(product);
this.data.SaveChanges();
return product.Id;
}
public int AddProductCategory(Category category)
{
this.data.Categories.Add(category);
this.data.SaveChanges();
return category.Id;
}
public bool CategoryExist(string name)
{
return this.data.Categories.All().Any(s => s.Name == name);
}
public IEnumerable<ProductListViewModel> GetLatestProducts(int? skip = 0, int? take = 10)
{
var products = data.Products.All()
......@@ -25,6 +44,11 @@ namespace MvcCoreTemplate.Services.Data
return products;
}
public int GetPagesCount(int? skip, int? take)
{
return this.data.Products.All().Count() / take.Value;
}
public ProductDetailsViewModel GetProductDetails(int id)
{
var product = this.data.Products.All().FirstOrDefault(p => p.Id == id);
......
......@@ -16,9 +16,12 @@ namespace MvcCoreTemplate.Web.Controllers
this.productsService = productsService;
}
public IActionResult Index(int? skip,int? take)
public IActionResult Index(int? skip=0,int? take=12)
{
var products = productsService.GetLatestProducts(skip, take);
ViewBag.LastTake = take;
ViewBag.PagesCount = productsService.GetPagesCount(skip, take);
ViewBag.CurrentPage = skip.Value / take.Value + 1; // ex. skip=20, take 12->20/12 = 1 | +1 = 2
return View(products);
}
}
......
......@@ -13,14 +13,10 @@
</PropertyGroup>
<ItemGroup>
<Compile Remove="Data\Migrations\**" />
<Content Remove="Data\Migrations\**" />
<EmbeddedResource Remove="Data\Migrations\**" />
<None Remove="Data\Migrations\**" />
</ItemGroup>
<ItemGroup>
<Compile Remove="Data\ApplicationDbContext.cs" />
<Compile Remove="Data\**" />
<Content Remove="Data\**" />
<EmbeddedResource Remove="Data\**" />
<None Remove="Data\**" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AutoMapper" Version="6.0.2" />
......@@ -48,7 +44,6 @@
<DotNetCliToolReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Tools" Version="1.0.0" />
</ItemGroup>
<ItemGroup>
<Folder Include="Data\" />
<Folder Include="Views\Features\" />
<Folder Include="Views\Products\" />
</ItemGroup>
......
......@@ -71,6 +71,7 @@ namespace MvcCoreTemplate.Web
services.AddTransient<IEmailSender, AuthMessageSender>();
services.AddTransient<ISmsSender, AuthMessageSender>();
services.AddTransient<ICommonService, CommonService>();
services.AddTransient<IProductService, ProductService>();
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
......
@model List<ProductListViewModel>
@{
ViewData["Title"] = "Get latest products";
Layout = "~/Views/Shared/_Layout.cshtml";
int counter = 0;
}
<h2>Index</h2>
<div class="container">
<br />
<div class="row">
@foreach (var product in Model)
{
if (counter != 0 && counter % 3 == 0)
{
@:</div>
@:<div class="row">
}
<div class="col-md-4">
<div class="thumbnail">
<img src="@product.ImageUrl" >
<div class="caption">
<h3 id="thumbnail-label">@product.Name</h3>
<p>$@product.Price</p>
</div>
</div>
</div>
counter++;
}
</div>
<br />
@if ((int)ViewBag.PagesCount>1)
{
<div class="row">
<nav aria-label="Page navigation">
<ul class="pagination">
@*<li>
<a href="#" aria-label="Previous">
<span aria-hidden="true">&laquo;</span>
</a>
</li>*@
@for (int i = 1; i <= (int)ViewBag.PagesCount; i++)
{
<li><a href="/Products/index?skip=@((i-1)*(int)ViewBag.LastTake)&take=@((int)ViewBag.LastTake)">@i</a></li>
}
@*<li>
<a href="#" aria-label="Next">
<span aria-hidden="true">&raquo;</span>
</a>
</li>*@
</ul>
</nav>
</div>
}
</div>
......@@ -33,7 +33,7 @@
<div class="navbar-collapse collapse">
<ul class="nav navbar-nav">
<li><a asp-area="" asp-controller="Home" asp-action="Index">Home</a></li>
<li><a asp-area="" asp-controller="Features" asp-action="Index">Features</a></li>
<li><a asp-area="" asp-controller="Products" asp-action="Index">Products example</a></li>
<li><a asp-area="" asp-controller="Features" asp-action="UnitOfWork">Features by Unit of work</a></li>
</ul>
@await Html.PartialAsync("_LoginPartial")
......
......@@ -3,4 +3,6 @@
@using MvcCoreTemplate.Web.Models.AccountViewModels
@using MvcCoreTemplate.Web.Models.ManageViewModels
@using Microsoft.AspNetCore.Identity
@using MvcCoreTemplate.Web.Infrastructure.Models.Products
@using MvcCoreTemplate.Web.Infrastructure.Models.Images
@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers
......@@ -27,6 +27,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MvcCoreTemplate.Data.Common
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MvcCoreTemplate.Data.Models", "MvcCoreTemplate.Data.Models\MvcCoreTemplate.Data.Models.csproj", "{F0286C24-B055-4C9C-97BE-13C0EBCB0031}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MvcCoreTemplate.Crawler", "MvcCoreTemplate.Crawler\MvcCoreTemplate.Crawler.csproj", "{0F567091-AC61-40E8-BD62-A98EDAF524D1}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
......@@ -65,6 +67,10 @@ Global
{F0286C24-B055-4C9C-97BE-13C0EBCB0031}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F0286C24-B055-4C9C-97BE-13C0EBCB0031}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F0286C24-B055-4C9C-97BE-13C0EBCB0031}.Release|Any CPU.Build.0 = Release|Any CPU
{0F567091-AC61-40E8-BD62-A98EDAF524D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0F567091-AC61-40E8-BD62-A98EDAF524D1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0F567091-AC61-40E8-BD62-A98EDAF524D1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0F567091-AC61-40E8-BD62-A98EDAF524D1}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
......@@ -78,5 +84,6 @@ Global
{61EEFBFD-1C6F-4F6B-9D2D-9D0C5BD5E71E} = {3A79ADCA-45DD-43CE-8D3E-4D5F10945C4E}
{EA1C820E-9705-48B2-AD18-7121FCAF651D} = {C19362E2-DE5D-410B-80E4-F7C949FFBD2B}
{F0286C24-B055-4C9C-97BE-13C0EBCB0031} = {C19362E2-DE5D-410B-80E4-F7C949FFBD2B}
{0F567091-AC61-40E8-BD62-A98EDAF524D1} = {880AD230-6FB7-4652-BFF3-B5EFA22F976B}
EndGlobalSection
EndGlobal
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment